From 457d804aa272637275fc8712ee6e4410d180d2a0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 19 Nov 2020 17:29:01 -0500 Subject: [PATCH] Add pipeline tests for AWS CloudTrail The tests revealed a few issues. There was an error in the pipeline for update-user-json.log because serviceEventDetails was not present. This was the error "error": { "message": "Cannot invoke \\\"Object.getClass()\\\" because \\\"receiver\\\" is null" } The aws.cloudtrail.read_only field was mapped as keyword but was actual a JSON boolean. I changed the type to boolean, but do not plan to backport this change to Filebeat. And lastly some ECS user_agent fields were missing. This depends on https://github.com/elastic/elastic-package/pull/177 to make the flattened fields pass test validation. --- packages/auditd/_dev/deploy/docker/Dockerfile | 5 - .../test/pipeline/add-user-to-group-json.log | 1 + .../add-user-to-group-json.log-config.json | 5 + .../add-user-to-group-json.log-expected.json | 69 +++++ .../_dev/test/pipeline/assume-role-json.log | 1 + .../pipeline/assume-role-json.log-config.json | 5 + .../assume-role-json.log-expected.json | 125 ++++++++++ .../test/pipeline/change-password-json.log | 2 + .../change-password-json.log-config.json | 5 + .../change-password-json.log-expected.json | 112 +++++++++ .../test/pipeline/cloudtrail-digest-json.log | 1 + .../cloudtrail-digest-json.log-config.json | 5 + .../cloudtrail-digest-json.log-expected.json | 12 + .../_dev/test/pipeline/console-login-json.log | 3 + .../console-login-json.log-config.json | 5 + .../console-login-json.log-expected.json | 235 ++++++++++++++++++ .../test/pipeline/create-access-key-json.log | 1 + .../create-access-key-json.log-config.json | 5 + .../create-access-key-json.log-expected.json | 81 ++++++ .../_dev/test/pipeline/create-group-json.log | 2 + .../create-group-json.log-config.json | 5 + .../create-group-json.log-expected.json | 138 ++++++++++ .../test/pipeline/create-key-pair-json.log | 1 + .../create-key-pair-json.log-config.json | 5 + .../create-key-pair-json.log-expected.json | 90 +++++++ .../_dev/test/pipeline/create-trail-json.log | 1 + .../create-trail-json.log-config.json | 5 + .../create-trail-json.log-expected.json | 78 ++++++ .../_dev/test/pipeline/create-user-json.log | 1 + .../pipeline/create-user-json.log-config.json | 5 + .../create-user-json.log-expected.json | 78 ++++++ .../create-virtual-mfa-device-json.log | 1 + ...te-virtual-mfa-device-json.log-config.json | 5 + ...-virtual-mfa-device-json.log-expected.json | 73 ++++++ .../pipeline/deactivate-mfa-device-json.log | 1 + ...deactivate-mfa-device-json.log-config.json | 5 + ...activate-mfa-device-json.log-expected.json | 73 ++++++ .../test/pipeline/delete-access-key-json.log | 1 + .../delete-access-key-json.log-config.json | 5 + .../delete-access-key-json.log-expected.json | 73 ++++++ .../_dev/test/pipeline/delete-bucket-json.log | 1 + .../delete-bucket-json.log-config.json | 5 + .../delete-bucket-json.log-expected.json | 77 ++++++ .../_dev/test/pipeline/delete-group-json.log | 2 + .../delete-group-json.log-config.json | 5 + .../delete-group-json.log-expected.json | 128 ++++++++++ .../pipeline/delete-ssh-public-key-json.log | 1 + ...delete-ssh-public-key-json.log-config.json | 5 + ...lete-ssh-public-key-json.log-expected.json | 73 ++++++ .../_dev/test/pipeline/delete-trail-json.log | 1 + .../delete-trail-json.log-config.json | 5 + .../delete-trail-json.log-expected.json | 58 +++++ .../_dev/test/pipeline/delete-user-json.log | 1 + .../pipeline/delete-user-json.log-config.json | 5 + .../delete-user-json.log-expected.json | 72 ++++++ .../delete-virtual-mfa-device-json.log | 1 + ...te-virtual-mfa-device-json.log-config.json | 5 + ...-virtual-mfa-device-json.log-expected.json | 67 +++++ .../test/pipeline/enable-mfa-device-json.log | 1 + .../enable-mfa-device-json.log-config.json | 5 + .../enable-mfa-device-json.log-expected.json | 72 ++++++ .../_dev/test/pipeline/insight-json.log | 1 + .../pipeline/insight-json.log-config.json | 5 + .../pipeline/insight-json.log-expected.json | 25 ++ .../pipeline/remove-user-from-group-json.log | 1 + ...emove-user-from-group-json.log-config.json | 5 + ...ove-user-from-group-json.log-expected.json | 73 ++++++ .../_dev/test/pipeline/start-logging-json.log | 1 + .../start-logging-json.log-config.json | 5 + .../start-logging-json.log-expected.json | 62 +++++ .../_dev/test/pipeline/stop-logging-json.log | 1 + .../stop-logging-json.log-config.json | 5 + .../stop-logging-json.log-expected.json | 62 +++++ .../test/pipeline/update-access-key-json.log | 1 + .../update-access-key-json.log-config.json | 5 + .../update-access-key-json.log-expected.json | 74 ++++++ .../update-accout-password-policy-json.log | 1 + ...ccout-password-policy-json.log-config.json | 5 + ...out-password-policy-json.log-expected.json | 72 ++++++ .../_dev/test/pipeline/update-group-json.log | 2 + .../update-group-json.log-config.json | 5 + .../update-group-json.log-expected.json | 126 ++++++++++ .../pipeline/update-login-profile-json.log | 1 + .../update-login-profile-json.log-config.json | 5 + ...pdate-login-profile-json.log-expected.json | 72 ++++++ .../pipeline/update-ssh-public-key-json.log | 2 + ...update-ssh-public-key-json.log-config.json | 5 + ...date-ssh-public-key-json.log-expected.json | 144 +++++++++++ .../_dev/test/pipeline/update-trail-json.log | 2 + .../update-trail-json.log-config.json | 5 + .../update-trail-json.log-expected.json | 155 ++++++++++++ .../_dev/test/pipeline/update-user-json.log | 2 + .../pipeline/update-user-json.log-config.json | 5 + .../update-user-json.log-expected.json | 78 ++++++ .../pipeline/upload-ssh-public-key-json.log | 1 + ...upload-ssh-public-key-json.log-config.json | 5 + ...load-ssh-public-key-json.log-expected.json | 78 ++++++ .../elasticsearch/ingest_pipeline/default.yml | 6 +- .../aws/data_stream/cloudtrail/fields/ecs.yml | 15 ++ .../data_stream/cloudtrail/fields/fields.yml | 2 +- packages/aws/manifest.yml | 2 +- 101 files changed, 3026 insertions(+), 10 deletions(-) delete mode 100644 packages/auditd/_dev/deploy/docker/Dockerfile create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-expected.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-config.json create mode 100644 packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-expected.json diff --git a/packages/auditd/_dev/deploy/docker/Dockerfile b/packages/auditd/_dev/deploy/docker/Dockerfile deleted file mode 100644 index 8ca5d93a93a7..000000000000 --- a/packages/auditd/_dev/deploy/docker/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM alpine - -COPY ./audit.log . - -ENTRYPOINT [ "/bin/sh" ] \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log new file mode 100644 index 000000000000..4c067668bedb --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-expected.json new file mode 100644 index 000000000000..5687a2a15fb5 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/add-user-to-group-json.log-expected.json @@ -0,0 +1,69 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2014-03-25T21:08:14.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.114840Z", + "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "AddUserToGroup", + "type": [ + "group", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.0", + "flattened": { + "request_parameters": { + "userName": "Bob", + "groupName": "admin" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "session_context": { + "mfa_authenticated": "false", + "creation_date": "2014-03-25T18:45:11.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "request_parameters": "{groupName=admin, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "AWSConsole" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log new file mode 100644 index 000000000000..c2a4a5e884bb --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIN5ATK5U7KEXAMPLE:JohnRole1","arn":"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1","accountId":"111111111111","accessKeyId":"AKIAI44QH8DHBEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-10-02T21:50:54Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIN5ATK5U7KEXAMPLE","arn":"arn:aws:iam::111111111111:role/JohnRole1","accountId":"111111111111","userName":"JohnDoe"}}},"eventTime":"2019-10-02T22:12:29Z","eventSource":"sts.amazonaws.com","eventName":"AssumeRole","awsRegion":"us-east-2","sourceIPAddress":"123.145.67.89","userAgent":"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239","requestParameters":{"incomingTransitiveTags":{"Department":"Engineering"},"tags":[{"value":"johndoe@example.com","key":"Email"},{"value":"12345","key":"CostCenter"}],"roleArn":"arn:aws:iam::111111111111:role/JohnRole2","roleSessionName":"Role2WithTags","transitiveTagKeys":["Email","CostCenter"],"durationSeconds":3600},"responseElements":{"credentials":{"accessKeyId":"ASIAWHOJDLGPOEXAMPLE","expiration":"Oct 2, 2019 11:12:29 PM","sessionToken":"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN"},"assumedRoleUser":{"assumedRoleId":"AROAIFR7WHDTSOYQYHFUE:Role2WithTags","arn":"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"}},"requestID":"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE","eventID":"1917948f-3042-46ec-98e2-62865EXAMPLE","resources":[{"ARN":"arn:aws:iam::111122223333:role/JohnRole2","accountId":"111111111111","type":"AWS::IAM::Role"}],"eventType":"AwsApiCall","recipientAccountId":"111111111111"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-expected.json new file mode 100644 index 000000000000..5feed30efb94 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/assume-role-json.log-expected.json @@ -0,0 +1,125 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "111111111111" + } + }, + "@timestamp": "2019-10-02T22:12:29.000Z", + "source": { + "geo": { + "continent_name": "Asia", + "region_iso_code": "CN-CQ", + "country_name": "China", + "region_name": "Chongqing", + "location": { + "lon": 106.5531, + "lat": 29.5569 + }, + "country_iso_code": "CN" + }, + "as": { + "number": 4837, + "organization": { + "name": "CHINA UNICOM China169 Backbone" + } + }, + "address": "123.145.67.89", + "ip": "123.145.67.89" + }, + "event": { + "ingested": "2020-11-19T22:16:17.142969600Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}", + "provider": "sts.amazonaws.com", + "kind": "event", + "action": "AssumeRole", + "id": "1917948f-3042-46ec-98e2-62865EXAMPLE", + "type": [ + "info" + ], + "category": [ + "authentication" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "incomingTransitiveTags": { + "Department": "Engineering" + }, + "transitiveTagKeys": [ + "Email", + "CostCenter" + ], + "durationSeconds": 3600, + "roleArn": "arn:aws:iam::111111111111:role/JohnRole2", + "roleSessionName": "Role2WithTags", + "tags": [ + { + "value": "johndoe@example.com", + "key": "Email" + }, + { + "value": "12345", + "key": "CostCenter" + } + ] + }, + "response_elements": { + "assumedRoleUser": { + "assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags", + "arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags" + }, + "credentials": { + "accessKeyId": "ASIAWHOJDLGPOEXAMPLE", + "sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN", + "expiration": "Oct 2, 2019 11:12:29 PM" + } + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "AKIAI44QH8DHBEXAMPLE", + "session_context": { + "mfa_authenticated": "false", + "session_issuer": { + "account_id": "111111111111", + "type": "Role", + "arn": "arn:aws:iam::111111111111:role/JohnRole1", + "principal_id": "AROAIN5ATK5U7KEXAMPLE" + }, + "creation_date": "2019-10-02T21:50:54.000Z" + }, + "type": "AssumedRole", + "arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1" + }, + "recipient_account_id": "111111111111", + "request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}", + "response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}" + } + }, + "user": { + "name": "JohnDoe", + "id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239", + "os": { + "name": "Linux", + "version": "4.9.184", + "full": "Linux 4.9.184" + }, + "device": { + "name": "Spider" + }, + "version": "1.16.248" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log new file mode 100644 index 000000000000..b3c1f2a10d31 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:09:33Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"AccessDeniedException","errorMessage":"An unknown error occurred","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE","eventID":"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:03:36Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5c16-4eda-9724-EXAMPLE","eventID":"EXAMPLE-35a7-4c25-9fc7-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-expected.json new file mode 100644 index 000000000000..a888654d3eca --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/change-password-json.log-expected.json @@ -0,0 +1,112 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T00:09:33.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.187739500Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:09:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"AccessDeniedException\",\"errorMessage\":\"An unknown error occurred\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE\",\"eventID\":\"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "ChangePassword", + "id": "EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "An unknown error occurred", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "error_code": "AccessDeniedException", + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + }, + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T00:03:36.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.187745300Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T00:03:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"ChangePassword\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":null,\"responseElements\":null,\"requestID\":\"EXAMPLE-5c16-4eda-9724-EXAMPLE\",\"eventID\":\"EXAMPLE-35a7-4c25-9fc7-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "ChangePassword", + "id": "EXAMPLE-35a7-4c25-9fc7-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log new file mode 100644 index 000000000000..f3393babceb9 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log @@ -0,0 +1 @@ +{"awsAccountId":"123456789123","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"alice-bucket","digestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"47aaa19f7eec22e9bd0b5e58cfade8cb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"alice-bucket","previousDigestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987","logFiles":[{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz","hashValue":"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:21:28Z","oldestEventTime":"2020-09-11T19:21:28Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz","hashValue":"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:51:21Z","oldestEventTime":"2020-09-11T18:51:21Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz","hashValue":"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:46:45Z","oldestEventTime":"2020-09-11T18:46:45Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz","hashValue":"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:01:06Z","oldestEventTime":"2020-09-11T19:01:06Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz","hashValue":"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:16:12Z","oldestEventTime":"2020-09-11T19:16:12Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz","hashValue":"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:56:05Z","oldestEventTime":"2020-09-11T18:56:05Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz","hashValue":"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:06:31Z","oldestEventTime":"2020-09-11T19:06:31Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz","hashValue":"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:41:58Z","oldestEventTime":"2020-09-11T18:41:58Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz","hashValue":"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:37:10Z","oldestEventTime":"2020-09-11T18:37:10Z"}]} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-expected.json new file mode 100644 index 000000000000..8207118c4cb7 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/cloudtrail-digest-json.log-expected.json @@ -0,0 +1,12 @@ +{ + "expected": [ + { + "event": { + "ingested": "2020-11-19T22:16:17.237931600Z", + "original": "{\"awsAccountId\":\"123456789123\",\"digestStartTime\":\"2020-09-11T18:36:49Z\",\"digestEndTime\":\"2020-09-11T19:36:49Z\",\"digestS3Bucket\":\"alice-bucket\",\"digestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz\",\"digestPublicKeyFingerprint\":\"47aaa19f7eec22e9bd0b5e58cfade8cb\",\"digestSignatureAlgorithm\":\"SHA256withRSA\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\",\"previousDigestS3Bucket\":\"alice-bucket\",\"previousDigestS3Object\":\"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz\",\"previousDigestHashValue\":\"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860\",\"previousDigestHashAlgorithm\":\"SHA-256\",\"previousDigestSignature\":\"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987\",\"logFiles\":[{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz\",\"hashValue\":\"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:26:24Z\",\"oldestEventTime\":\"2020-09-11T19:26:24Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz\",\"hashValue\":\"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:11:18Z\",\"oldestEventTime\":\"2020-09-11T19:11:18Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz\",\"hashValue\":\"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:32:04Z\",\"oldestEventTime\":\"2020-09-11T18:32:04Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1925Z_zJNGzQovyNAImZV9.json.gz\",\"hashValue\":\"45a2906f55cbfc912584e9425f8d3d8d6fabf571a45a5ecd7d2a0f4132b81689\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:21:28Z\",\"oldestEventTime\":\"2020-09-11T19:21:28Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1855Z_RqN9YzoKAJCKbejj.json.gz\",\"hashValue\":\"515cc8be750d815266b4fc799c7600765f22502d29f5bb9d5c8969ffc5ab7097\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:51:21Z\",\"oldestEventTime\":\"2020-09-11T18:51:21Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1850Z_jLldN7U8XrspES8p.json.gz\",\"hashValue\":\"18650414e79e084dff02da66253f071347f7bb5c4863279bafe7762a980f7c0b\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:46:45Z\",\"oldestEventTime\":\"2020-09-11T18:46:45Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1905Z_jBNdmg4bSGxZ3wC8.json.gz\",\"hashValue\":\"54050ec665636f1985f5b51ae43c74a58282cb2e500492a45f20a4dc1bf8a6d5\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:01:06Z\",\"oldestEventTime\":\"2020-09-11T19:01:06Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1920Z_bj5DRrmILF6jK23a.json.gz\",\"hashValue\":\"6e0d8fcbd712d3f6d1caf4a872681f4290b05ed8a8f1c9450a0a6db92ccab4d7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:16:12Z\",\"oldestEventTime\":\"2020-09-11T19:16:12Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1900Z_6LjrkrhsLQMzCiSN.json.gz\",\"hashValue\":\"b2b0e2804d1c6b92d76eee203d7eba32d3d003e6967f175723a83ecc2d7ad4ba\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:56:05Z\",\"oldestEventTime\":\"2020-09-11T18:56:05Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1910Z_DLyqye8LaeoD204N.json.gz\",\"hashValue\":\"4397a13565a67d9ed6e57737b98eb7e61ca52bb191c9b5da0423136dfc5581c7\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T19:06:31Z\",\"oldestEventTime\":\"2020-09-11T19:06:31Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1845Z_TSDKyASOn2ejOq5n.json.gz\",\"hashValue\":\"94f09d2398632c7b0c0066ed5d56768632dd2e06ed9c80af9d0c2c5f59bd60b6\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:41:58Z\",\"oldestEventTime\":\"2020-09-11T18:41:58Z\"},{\"s3Bucket\":\"alice-bucket\",\"s3Object\":\"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1840Z_btJydJ2t7hCRnjsN.json.gz\",\"hashValue\":\"9044f9a05d70688bc6f6048d5f8d00764ab65e132b8ffefb193b22ca4394d771\",\"hashAlgorithm\":\"SHA-256\",\"newestEventTime\":\"2020-09-11T18:37:10Z\",\"oldestEventTime\":\"2020-09-11T18:37:10Z\"}]}", + "type": "info", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log new file mode 100644 index 000000000000..14fb436a9389 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log @@ -0,0 +1,3 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JohnDoe","accountId":"111122223333","userName":"JohnDoe"},"eventTime":"2014-07-16T15:49:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.110","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","requestParameters":null,"responseElements":{"ConsoleLogin":"Success"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/s3/","MFAUsed":"No"},"eventID":"3fcfb182-98f8-4744-bd45-10aEXAMPLE"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"AIDACKCEVSQ6C2EXAMPLE","arn":"arn:aws:iam::111122223333:user/JaneDoe","accountId":"111122223333","userName":"JaneDoe"},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} +{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName","arn":"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName","accountId":"123456789012","accessKeyId":"AKIAIOSFODNN7EXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"20131102T010628Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIDPPEZS35WEXAMPLE","arn":"arn:aws:iam::123456789012:role/RoleToBeAssumed","accountId":"123456789012","userName":"RoleToBeAssumed"}}},"eventTime":"2014-07-08T17:35:27Z","eventSource":"signin.amazonaws.com","eventName":"ConsoleLogin","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.100","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0","errorMessage":"Failed authentication","requestParameters":null,"responseElements":{"ConsoleLogin":"Failure"},"additionalEventData":{"MobileVersion":"No","LoginTo":"https://console.aws.amazon.com/sns","MFAUsed":"No"},"eventID":"11ea990b-4678-4bcd-8fbe-625EXAMPLE"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-expected.json new file mode 100644 index 000000000000..bce9f5b0e2b8 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/console-login-json.log-expected.json @@ -0,0 +1,235 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "111122223333" + } + }, + "@timestamp": "2014-07-16T15:49:27.000Z", + "source": { + "address": "192.0.2.110", + "ip": "192.0.2.110" + }, + "event": { + "ingested": "2020-11-19T22:16:17.251357800Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JohnDoe\",\"accountId\":\"111122223333\",\"userName\":\"JohnDoe\"},\"eventTime\":\"2014-07-16T15:49:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.110\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Success\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/s3/\",\"MFAUsed\":\"No\"},\"eventID\":\"3fcfb182-98f8-4744-bd45-10aEXAMPLE\"}", + "provider": "signin.amazonaws.com", + "kind": "event", + "action": "ConsoleLogin", + "id": "3fcfb182-98f8-4744-bd45-10aEXAMPLE", + "type": [ + "info" + ], + "category": [ + "authentication" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "additional_eventdata": { + "LoginTo": "https://console.aws.amazon.com/s3/", + "MobileVersion": "No", + "MFAUsed": "No" + }, + "response_elements": { + "ConsoleLogin": "Success" + } + }, + "user_identity": { + "type": "IAMUser", + "arn": "arn:aws:iam::111122223333:user/JohnDoe" + }, + "additional_eventdata": "{LoginTo=https://console.aws.amazon.com/s3/, MobileVersion=No, MFAUsed=No}", + "console_login": { + "additional_eventdata": { + "login_to": "https://console.aws.amazon.com/s3/", + "mobile_version": false, + "mfa_used": false + } + }, + "response_elements": "{ConsoleLogin=Success}" + } + }, + "user": { + "name": "JohnDoe", + "id": "AIDACKCEVSQ6C2EXAMPLE" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "24.0." + } + }, + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "111122223333" + } + }, + "@timestamp": "2014-07-08T17:35:27.000Z", + "source": { + "address": "192.0.2.100", + "ip": "192.0.2.100" + }, + "event": { + "ingested": "2020-11-19T22:16:17.251366800Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"AIDACKCEVSQ6C2EXAMPLE\",\"arn\":\"arn:aws:iam::111122223333:user/JaneDoe\",\"accountId\":\"111122223333\",\"userName\":\"JaneDoe\"},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", + "provider": "signin.amazonaws.com", + "kind": "event", + "action": "ConsoleLogin", + "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", + "type": [ + "info" + ], + "category": [ + "authentication" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "Failed authentication", + "flattened": { + "additional_eventdata": { + "LoginTo": "https://console.aws.amazon.com/sns", + "MobileVersion": "No", + "MFAUsed": "No" + }, + "response_elements": { + "ConsoleLogin": "Failure" + } + }, + "additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}", + "console_login": { + "additional_eventdata": { + "login_to": "https://console.aws.amazon.com/sns", + "mobile_version": false, + "mfa_used": false + } + }, + "user_identity": { + "type": "IAMUser", + "arn": "arn:aws:iam::111122223333:user/JaneDoe" + }, + "response_elements": "{ConsoleLogin=Failure}" + } + }, + "user": { + "name": "JaneDoe", + "id": "AIDACKCEVSQ6C2EXAMPLE" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "24.0." + } + }, + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2014-07-08T17:35:27.000Z", + "source": { + "address": "192.0.2.100", + "ip": "192.0.2.100" + }, + "event": { + "ingested": "2020-11-19T22:16:17.251419800Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName\",\"arn\":\"arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName\",\"accountId\":\"123456789012\",\"accessKeyId\":\"AKIAIOSFODNN7EXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"20131102T010628Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIDPPEZS35WEXAMPLE\",\"arn\":\"arn:aws:iam::123456789012:role/RoleToBeAssumed\",\"accountId\":\"123456789012\",\"userName\":\"RoleToBeAssumed\"}}},\"eventTime\":\"2014-07-08T17:35:27Z\",\"eventSource\":\"signin.amazonaws.com\",\"eventName\":\"ConsoleLogin\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.100\",\"userAgent\":\"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0\",\"errorMessage\":\"Failed authentication\",\"requestParameters\":null,\"responseElements\":{\"ConsoleLogin\":\"Failure\"},\"additionalEventData\":{\"MobileVersion\":\"No\",\"LoginTo\":\"https://console.aws.amazon.com/sns\",\"MFAUsed\":\"No\"},\"eventID\":\"11ea990b-4678-4bcd-8fbe-625EXAMPLE\"}", + "provider": "signin.amazonaws.com", + "kind": "event", + "action": "ConsoleLogin", + "id": "11ea990b-4678-4bcd-8fbe-625EXAMPLE", + "type": [ + "info" + ], + "category": [ + "authentication" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "Failed authentication", + "flattened": { + "additional_eventdata": { + "LoginTo": "https://console.aws.amazon.com/sns", + "MobileVersion": "No", + "MFAUsed": "No" + }, + "response_elements": { + "ConsoleLogin": "Failure" + } + }, + "additional_eventdata": "{LoginTo=https://console.aws.amazon.com/sns, MobileVersion=No, MFAUsed=No}", + "console_login": { + "additional_eventdata": { + "login_to": "https://console.aws.amazon.com/sns", + "mobile_version": false, + "mfa_used": false + } + }, + "user_identity": { + "access_key_id": "AKIAIOSFODNN7EXAMPLE", + "session_context": { + "mfa_authenticated": "false", + "session_issuer": { + "account_id": "123456789012", + "type": "Role", + "arn": "arn:aws:iam::123456789012:role/RoleToBeAssumed", + "principal_id": "AROAIDPPEZS35WEXAMPLE" + } + }, + "type": "AssumedRole", + "arn": "arn:aws:sts::123456789012:assumed-role/RoleToBeAssumed/MySessionName" + }, + "response_elements": "{ConsoleLogin=Failure}" + } + }, + "user": { + "name": "RoleToBeAssumed", + "id": "AROAIDPPEZS35WEXAMPLE:AssumedRoleSessionName" + }, + "user_agent": { + "name": "Firefox", + "original": "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", + "os": { + "name": "Windows", + "version": "7", + "full": "Windows 7" + }, + "device": { + "name": "Other" + }, + "version": "24.0." + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log new file mode 100644 index 000000000000..d18fcffb9336 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T20:43:06Z","eventSource":"iam.amazonaws.com","eventName":"CreateAccessKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"userName":"Bob"},"responseElements":{"accessKey":{"accessKeyId":"EXAMPLE_KEY_ID","status":"Active","userName":"Bob","createDate":"Jan 8, 2020 8:43:06 PM"}},"requestID":"EXAMPLE-823a-48dc-8fa9-EXAMPLE","eventID":"EXAMPLE-3cab-40f8-938b-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-expected.json new file mode 100644 index 000000000000..5920c8034ae5 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-access-key-json.log-expected.json @@ -0,0 +1,81 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-08T20:43:06.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.351340900Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:43:06Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"accessKey\":{\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"status\":\"Active\",\"userName\":\"Bob\",\"createDate\":\"Jan 8, 2020 8:43:06 PM\"}},\"requestID\":\"EXAMPLE-823a-48dc-8fa9-EXAMPLE\",\"eventID\":\"EXAMPLE-3cab-40f8-938b-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "CreateAccessKey", + "id": "EXAMPLE-3cab-40f8-938b-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob" + }, + "response_elements": { + "accessKey": { + "accessKeyId": "EXAMPLE_KEY_ID", + "userName": "Bob", + "status": "Active", + "createDate": "Jan 8, 2020 8:43:06 PM" + } + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{userName=Bob}", + "response_elements": "{accessKey={accessKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Active, createDate=Jan 8, 2020 8:43:06 PM}}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log new file mode 100644 index 000000000000..f46f6d474c6a --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-09T01:48:44Z","eventSource":"iam.amazonaws.com","eventName":"CreateGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"groupName":"TEST-GROUP"},"responseElements":{"group":{"createDate":"Jan 9, 2020 1:48:44 AM","path":"/","arn":"arn:aws:iam::0123456789012:group/TEST-GROUP","groupName":"TEST-GROUP","groupId":"EXAMPLE_ID"}},"requestID":"EXAMPLE-769d-4a61-b731-EXAMPLE","eventID":"EXAMPLE-37ec-425a-a7ef-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T02:22:03Z","eventSource":"iam.amazonaws.com","eventName":"CreateGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"EntityAlreadyExistsException","errorMessage":"Group with name TEST-GROUP already exists.","requestParameters":{"groupName":"TEST-GROUP"},"responseElements":null,"requestID":"EXAMPLE-c8ae-44dc-8114-EXAMPLE","eventID":"EXAMPLE-09c6-4745-af70-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-expected.json new file mode 100644 index 000000000000..365d5ea0ed33 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-group-json.log-expected.json @@ -0,0 +1,138 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T01:48:44.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.399618200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T01:48:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":{\"group\":{\"createDate\":\"Jan 9, 2020 1:48:44 AM\",\"path\":\"/\",\"arn\":\"arn:aws:iam::0123456789012:group/TEST-GROUP\",\"groupName\":\"TEST-GROUP\",\"groupId\":\"EXAMPLE_ID\"}},\"requestID\":\"EXAMPLE-769d-4a61-b731-EXAMPLE\",\"eventID\":\"EXAMPLE-37ec-425a-a7ef-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "CreateGroup", + "id": "EXAMPLE-37ec-425a-a7ef-EXAMPLE", + "type": [ + "group", + "creation" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP" + }, + "response_elements": { + "group": { + "path": "/", + "groupName": "TEST-GROUP", + "arn": "arn:aws:iam::0123456789012:group/TEST-GROUP", + "groupId": "EXAMPLE_ID", + "createDate": "Jan 9, 2020 1:48:44 AM" + } + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP}", + "response_elements": "{group={path=/, groupName=TEST-GROUP, groupId=EXAMPLE_ID, arn=arn:aws:iam::0123456789012:group/TEST-GROUP, createDate=Jan 9, 2020 1:48:44 AM}}" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + }, + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T02:22:03.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.399623700Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:22:03Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c8ae-44dc-8114-EXAMPLE\",\"eventID\":\"EXAMPLE-09c6-4745-af70-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "CreateGroup", + "id": "EXAMPLE-09c6-4745-af70-EXAMPLE", + "type": [ + "group", + "creation" + ], + "category": [ + "iam" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "Group with name TEST-GROUP already exists.", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP" + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "error_code": "EntityAlreadyExistsException", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP}" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log new file mode 100644 index 000000000000..5b9c40ad40c4 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-06T15:15:06Z"}}},"eventTime":"2014-03-06T17:10:34Z","eventSource":"ec2.amazonaws.com","eventName":"CreateKeyPair","awsRegion":"us-east-2","sourceIPAddress":"72.21.198.64","userAgent":"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx","requestParameters":{"keyName":"mykeypair"},"responseElements":{"keyName":"mykeypair","keyFingerprint":"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21","keyMaterial":""}} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-expected.json new file mode 100644 index 000000000000..87f5636a7a9d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-key-pair-json.log-expected.json @@ -0,0 +1,90 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2014-03-06T17:10:34.000Z", + "source": { + "geo": { + "continent_name": "North America", + "region_iso_code": "US-VA", + "city_name": "Ashburn", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Virginia", + "location": { + "lon": -77.4728, + "lat": 39.0481 + } + }, + "as": { + "number": 16509, + "organization": { + "name": "Amazon.com, Inc." + } + }, + "address": "72.21.198.64", + "ip": "72.21.198.64" + }, + "event": { + "ingested": "2020-11-19T22:16:17.471316700Z", + "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-06T15:15:06Z\"}}},\"eventTime\":\"2014-03-06T17:10:34Z\",\"eventSource\":\"ec2.amazonaws.com\",\"eventName\":\"CreateKeyPair\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"72.21.198.64\",\"userAgent\":\"EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx\",\"requestParameters\":{\"keyName\":\"mykeypair\"},\"responseElements\":{\"keyName\":\"mykeypair\",\"keyFingerprint\":\"30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21\",\"keyMaterial\":\"\u003csensitiveDataRemoved\u003e\"}}", + "provider": "ec2.amazonaws.com", + "kind": "event", + "action": "CreateKeyPair", + "type": [ + "admin", + "creation" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.0", + "flattened": { + "request_parameters": { + "keyName": "mykeypair" + }, + "response_elements": { + "keyMaterial": "\u003csensitiveDataRemoved\u003e", + "keyName": "mykeypair", + "keyFingerprint": "30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "session_context": { + "mfa_authenticated": "false", + "creation_date": "2014-03-06T15:15:06.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "request_parameters": "{keyName=mykeypair}", + "response_elements": "{keyMaterial=\u003csensitiveDataRemoved\u003e, keyFingerprint=30:1d:46:d0:5b:ad:7e:1b:b6:70:62:8b:ff:38:b5:e9:ab:5d:b8:21, keyName=mykeypair}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "Other", + "original": "EC2ConsoleBackend, aws-sdk-java/Linux/x.xx.fleetxen Java_HotSpot(TM)_64-Bit_Server_VM/xx", + "os": { + "name": "Linux" + }, + "device": { + "name": "Other" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log new file mode 100644 index 000000000000..ebc0c708b042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"sessionIssuer":{},"webIdFederationData":{},"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T15:30:25Z","eventSource":"cloudtrail.amazonaws.com","eventName":"CreateTrail","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"name":"TEST-trail","s3BucketName":"TEST-cloudtrail-bucket","includeGlobalServiceEvents":true,"isMultiRegionTrail":true,"enableLogFileValidation":true,"kmsKeyId":"","isOrganizationTrail":false},"responseElements":{"name":"TEST-trail","s3BucketName":"TEST-cloudtrail-bucket","includeGlobalServiceEvents":true,"isMultiRegionTrail":true,"trailARN":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail","logFileValidationEnabled":true,"isOrganizationTrail":false},"requestID":"EXAMPLE-5149-4cf2-be99-EXAMPLE","eventID":"EXAMPLE-d04b-4eff-833a-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-expected.json new file mode 100644 index 000000000000..27a8e54faf1d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-trail-json.log-expected.json @@ -0,0 +1,78 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-west-2", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-08T15:30:25.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.524325600Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"CreateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"enableLogFileValidation\":true,\"kmsKeyId\":\"\",\"isOrganizationTrail\":false},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"TEST-cloudtrail-bucket\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":true,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-5149-4cf2-be99-EXAMPLE\",\"eventID\":\"EXAMPLE-d04b-4eff-833a-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "CreateTrail", + "id": "EXAMPLE-d04b-4eff-833a-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "isMultiRegionTrail": true, + "s3BucketName": "TEST-cloudtrail-bucket", + "name": "TEST-trail", + "enableLogFileValidation": true, + "kmsKeyId": "", + "isOrganizationTrail": false, + "includeGlobalServiceEvents": true + }, + "response_elements": { + "logFileValidationEnabled": true, + "isMultiRegionTrail": true, + "s3BucketName": "TEST-cloudtrail-bucket", + "name": "TEST-trail", + "trailARN": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail", + "isOrganizationTrail": false, + "includeGlobalServiceEvents": true + } + }, + "event_type": "AwsApiCall", + "read_only": false, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, enableLogFileValidation=true, kmsKeyId=, isOrganizationTrail=false, includeGlobalServiceEvents=true}", + "response_elements": "{logFileValidationEnabled=true, isMultiRegionTrail=true, s3BucketName=TEST-cloudtrail-bucket, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log new file mode 100644 index 000000000000..37e60f3f86cb --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2014-03-24T21:11:59Z","eventSource":"iam.amazonaws.com","eventName":"CreateUser","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.3.2 Python/2.7.5 Windows/7","requestParameters":{"userName":"Bob"},"responseElements":{"user":{"createDate":"Mar 24, 2014 9:11:59 PM","userName":"Bob","arn":"arn:aws:iam::123456789012:user/Bob","path":"/","userId":"EXAMPLEUSERID"}}} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-expected.json new file mode 100644 index 000000000000..a3822f1ea3ff --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-user-json.log-expected.json @@ -0,0 +1,78 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2014-03-24T21:11:59.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.562386700Z", + "original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2014-03-24T21:11:59Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateUser\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.3.2 Python/2.7.5 Windows/7\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":{\"user\":{\"createDate\":\"Mar 24, 2014 9:11:59 PM\",\"userName\":\"Bob\",\"arn\":\"arn:aws:iam::123456789012:user/Bob\",\"path\":\"/\",\"userId\":\"EXAMPLEUSERID\"}}}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "CreateUser", + "type": [ + "user", + "creation" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.0", + "flattened": { + "request_parameters": { + "userName": "Bob" + }, + "response_elements": { + "user": { + "path": "/", + "userName": "Bob", + "arn": "arn:aws:iam::123456789012:user/Bob", + "userId": "EXAMPLEUSERID", + "createDate": "Mar 24, 2014 9:11:59 PM" + } + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "request_parameters": "{userName=Bob}", + "response_elements": "{user={path=/, userName=Bob, arn=arn:aws:iam::123456789012:user/Bob, userId=EXAMPLEUSERID, createDate=Mar 24, 2014 9:11:59 PM}}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.3.2 Python/2.7.5 Windows/7", + "os": { + "name": "Windows" + }, + "device": { + "name": "Other" + }, + "version": "1.3.2" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log new file mode 100644 index 000000000000..5d33cd1ae3d7 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-11-27T15:07:22Z"}}},"eventTime":"2019-11-27T15:10:15Z","eventSource":"iam.amazonaws.com","eventName":"CreateVirtualMFADevice","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"console.amazonaws.com","requestParameters":{"virtualMFADeviceName":"Alice","path":"/"},"responseElements":{"virtualMFADevice":{"serialNumber":"arn:aws:iam::0123456789012:mfa/Alice"}},"requestID":"EXAMPLE-303b-4b0e-a8c7-EXAMPLE","eventID":"EXAMPLE-351c-472a-b089-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-expected.json new file mode 100644 index 000000000000..9c8a74abc34d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/create-virtual-mfa-device-json.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2019-11-27T15:10:15.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.596968500Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:10:15Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"CreateVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"virtualMFADeviceName\":\"Alice\",\"path\":\"/\"},\"responseElements\":{\"virtualMFADevice\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"}},\"requestID\":\"EXAMPLE-303b-4b0e-a8c7-EXAMPLE\",\"eventID\":\"EXAMPLE-351c-472a-b089-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "CreateVirtualMFADevice", + "id": "EXAMPLE-351c-472a-b089-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "path": "/", + "virtualMFADeviceName": "Alice" + }, + "response_elements": { + "virtualMFADevice": { + "serialNumber": "arn:aws:iam::0123456789012:mfa/Alice" + } + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "session_context": { + "mfa_authenticated": "false", + "creation_date": "2019-11-27T15:07:22.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{path=/, virtualMFADeviceName=Alice}", + "response_elements": "{virtualMFADevice={serialNumber=arn:aws:iam::0123456789012:mfa/Alice}}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "console.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log new file mode 100644 index 000000000000..bc8b0627f2ff --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-09T16:36:17Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T00:34:02Z","eventSource":"iam.amazonaws.com","eventName":"DeactivateMFADevice","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"userName":"Alice","serialNumber":"arn:aws:iam::0123456789012:mfa/Alice"},"responseElements":null,"requestID":"EXAMPLE-801a-4624-8fa0-EXAMPLE","eventID":"EXAMPLE-1889-416b-ace9-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-expected.json new file mode 100644 index 000000000000..bf3383711f80 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/deactivate-mfa-device-json.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T00:34:02.000Z", + "related": { + "user": [ + "Alice" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.634634200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeactivateMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Alice\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-801a-4624-8fa0-EXAMPLE\",\"eventID\":\"EXAMPLE-1889-416b-ace9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeactivateMFADevice", + "id": "EXAMPLE-1889-416b-ace9-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Alice", + "serialNumber": "arn:aws:iam::0123456789012:mfa/Alice" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-09T16:36:17.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{serialNumber=arn:aws:iam::0123456789012:mfa/Alice, userName=Alice}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log new file mode 100644 index 000000000000..63799766f5c6 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T19:09:36Z","eventSource":"iam.amazonaws.com","eventName":"DeleteAccessKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"userName":"Bob","accessKeyId":"EXAMPLE_ID"},"responseElements":null,"requestID":"EXAMPLE-3bea-41fa-a0b4-EXAMPLE","eventID":"EXAMPLE-0698-46bd-998d-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-expected.json new file mode 100644 index 000000000000..960c466c20e2 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-access-key-json.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-08T19:09:36.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.666712100Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T19:09:36Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"accessKeyId\":\"EXAMPLE_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-3bea-41fa-a0b4-EXAMPLE\",\"eventID\":\"EXAMPLE-0698-46bd-998d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteAccessKey", + "id": "EXAMPLE-0698-46bd-998d-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "accessKeyId": "EXAMPLE_ID", + "userName": "Bob" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{accessKeyId=EXAMPLE_ID, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log new file mode 100644 index 000000000000..913b109d7c0d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.04","userIdentity":{"type":"AssumedRole","principalId":"AIDAQRSTUVWXYZEXAMPLE:devdsk","arn":"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk","accountId":"777788889999","accessKeyId":"AKIAQRSTUVWXYZEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2016-11-14T17:25:26Z"},"sessionIssuer":{"type":"Role","principalId":"AIDAQRSTUVWXYZEXAMPLE","arn":"arn:aws:iam::777788889999:role/AssumeNothing","accountId":"777788889999","userName":"AssumeNothing"}}},"eventTime":"2016-11-14T17:25:45Z","eventSource":"s3.amazonaws.com","eventName":"DeleteBucket","awsRegion":"us-east-2","sourceIPAddress":"192.0.2.1","userAgent":"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]","requestParameters":{"bucketName":"my-test-bucket-cross-account"},"responseElements":null,"requestID":"EXAMPLE463D56D4C","eventID":"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"777788889999"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-expected.json new file mode 100644 index 000000000000..17b6d73ff16d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-bucket-json.log-expected.json @@ -0,0 +1,77 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "777788889999" + } + }, + "@timestamp": "2016-11-14T17:25:45.000Z", + "source": { + "address": "192.0.2.1", + "ip": "192.0.2.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.699662100Z", + "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE:devdsk\",\"arn\":\"arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk\",\"accountId\":\"777788889999\",\"accessKeyId\":\"AKIAQRSTUVWXYZEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2016-11-14T17:25:26Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AIDAQRSTUVWXYZEXAMPLE\",\"arn\":\"arn:aws:iam::777788889999:role/AssumeNothing\",\"accountId\":\"777788889999\",\"userName\":\"AssumeNothing\"}}},\"eventTime\":\"2016-11-14T17:25:45Z\",\"eventSource\":\"s3.amazonaws.com\",\"eventName\":\"DeleteBucket\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"192.0.2.1\",\"userAgent\":\"[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]\",\"requestParameters\":{\"bucketName\":\"my-test-bucket-cross-account\"},\"responseElements\":null,\"requestID\":\"EXAMPLE463D56D4C\",\"eventID\":\"dEXAMPLE-265a-41e0-9352-4401bEXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"777788889999\"}", + "provider": "s3.amazonaws.com", + "kind": "event", + "action": "DeleteBucket", + "id": "dEXAMPLE-265a-41e0-9352-4401bEXAMPLE", + "type": [ + "deletion" + ], + "category": [ + "file" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.04", + "flattened": { + "request_parameters": { + "bucketName": "my-test-bucket-cross-account" + } + }, + "user_identity": { + "access_key_id": "AKIAQRSTUVWXYZEXAMPLE", + "session_context": { + "mfa_authenticated": "false", + "session_issuer": { + "account_id": "777788889999", + "type": "Role", + "arn": "arn:aws:iam::777788889999:role/AssumeNothing", + "principal_id": "AIDAQRSTUVWXYZEXAMPLE" + }, + "creation_date": "2016-11-14T17:25:26.000Z" + }, + "type": "AssumedRole", + "arn": "arn:aws:sts::777788889999:assumed-role/AssumeNothing/devdsk" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "777788889999", + "request_parameters": "{bucketName=my-test-bucket-cross-account}" + } + }, + "user": { + "name": "AssumeNothing", + "id": "AIDAQRSTUVWXYZEXAMPLE:devdsk" + }, + "user_agent": { + "name": "aws-cli", + "original": "[aws-cli/1.11.10 Python/2.7.8 Linux/3.2.45-0.6.wd.865.49.315.metal1.x86_64 botocore/1.4.67]", + "os": { + "name": "Linux", + "version": "3.2.45", + "full": "Linux 3.2.45" + }, + "device": { + "name": "Spider" + }, + "version": "1.11.10" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log new file mode 100644 index 000000000000..97e75c9ab077 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-09T02:25:44Z","eventSource":"iam.amazonaws.com","eventName":"DeleteGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"groupName":"TEST-GROUP"},"responseElements":null,"requestID":"EXAMPLE-66cb-4775-a203-EXAMPLE","eventID":"EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_PRINCIPLE","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2020-01-09T02:25:11Z","eventSource":"iam.amazonaws.com","eventName":"DeleteGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"DeleteConflictException","errorMessage":"Cannot delete entity, must detach all policies first.","requestParameters":{"groupName":"TEST-GROUP"},"responseElements":null,"requestID":"EXAMPLE-2a3c-4a94-b24f-EXAMPLE","eventID":"EXAMPLE-5aa2-4b5f-a52a-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-expected.json new file mode 100644 index 000000000000..ea8e9dd01b02 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-group-json.log-expected.json @@ -0,0 +1,128 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T02:25:44.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.735300400Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T02:25:44Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-66cb-4775-a203-EXAMPLE\",\"eventID\":\"EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteGroup", + "id": "EXAMPLE-cbc2-4cc3-8bbc-EXAMPLE", + "type": [ + "group", + "deletion" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP}" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + }, + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T02:25:11.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.735308800Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_PRINCIPLE\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:25:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"DeleteConflictException\",\"errorMessage\":\"Cannot delete entity, must detach all policies first.\",\"requestParameters\":{\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-2a3c-4a94-b24f-EXAMPLE\",\"eventID\":\"EXAMPLE-5aa2-4b5f-a52a-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteGroup", + "id": "EXAMPLE-5aa2-4b5f-a52a-EXAMPLE", + "type": [ + "group", + "deletion" + ], + "category": [ + "iam" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "Cannot delete entity, must detach all policies first.", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP" + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "error_code": "DeleteConflictException", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_PRINCIPLE" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log new file mode 100644 index 000000000000..47451dfe3714 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:07:08Z","eventSource":"iam.amazonaws.com","eventName":"DeleteSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"sSHPublicKeyId":"EXAMPLE_KEY_ID","userName":"Bob"},"responseElements":null,"requestID":"EXAMPLE-7b34-44ae-a22f-EXAMPLE","eventID":"EXAMPLE-72ff-4d4f-9a8d-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-expected.json new file mode 100644 index 000000000000..37dcc108d478 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-ssh-public-key-json.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T16:07:08.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.790009400Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:07:08Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7b34-44ae-a22f-EXAMPLE\",\"eventID\":\"EXAMPLE-72ff-4d4f-9a8d-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteSSHPublicKey", + "id": "EXAMPLE-72ff-4d4f-9a8d-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "sSHPublicKeyId": "EXAMPLE_KEY_ID" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{sSHPublicKeyId=EXAMPLE_KEY_ID, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log new file mode 100644 index 000000000000..f747ff2c14a8 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T20:09:51Z","eventSource":"cloudtrail.amazonaws.com","eventName":"DeleteTrail","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":{"name":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail"},"responseElements":null,"requestID":"EXAMPLE-d44f-4a2a-966f-EXAMPLE","eventID":"EXAMPLE-3f9d-4634-8ff1-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-expected.json new file mode 100644 index 000000000000..559f6a2848a4 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-trail-json.log-expected.json @@ -0,0 +1,58 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-west-2", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T20:09:51.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.823102300Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T20:09:51Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"DeleteTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-d44f-4a2a-966f-EXAMPLE\",\"eventID\":\"EXAMPLE-3f9d-4634-8ff1-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "DeleteTrail", + "id": "EXAMPLE-3f9d-4634-8ff1-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "name": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail" + } + }, + "event_type": "AwsApiCall", + "read_only": false, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{name=arn:aws:cloudtrail:us-west-2:0123456789012:trail/test-trail}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log new file mode 100644 index 000000000000..ce00f5a11855 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-03T15:26:38Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-03T15:50:52Z","eventSource":"iam.amazonaws.com","eventName":"DeleteUser","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"userName":"Bob"},"responseElements":null,"requestID":"0e794d53-cdb5-4f7d-b7db-5EXAMPLE","eventID":"b89eb34b-8fcb-4cba-8439-d4EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-expected.json new file mode 100644 index 000000000000..30b528e220f1 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-user-json.log-expected.json @@ -0,0 +1,72 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2020-01-03T15:50:52.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.852239100Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-03T15:26:38Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-03T15:50:52Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"0e794d53-cdb5-4f7d-b7db-5EXAMPLE\",\"eventID\":\"b89eb34b-8fcb-4cba-8439-d4EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteUser", + "id": "b89eb34b-8fcb-4cba-8439-d4EXAMPLE", + "type": [ + "user", + "deletion" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-03T15:26:38.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "123456789012", + "request_parameters": "{userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log new file mode 100644 index 000000000000..ad22f516894c --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-09T16:36:17Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T00:34:02Z","eventSource":"iam.amazonaws.com","eventName":"DeleteVirtualMFADevice","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"serialNumber":"arn:aws:iam::0123456789012:mfa/Alice"},"responseElements":null,"requestID":"EXAMPLE-af91-4d1a-aaf2-EXAMPLE","eventID":"EXAMPLE-f8e6-4d5f-8525-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-expected.json new file mode 100644 index 000000000000..90153a42505c --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/delete-virtual-mfa-device-json.log-expected.json @@ -0,0 +1,67 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T00:34:02.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.885251200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T00:34:02Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"DeleteVirtualMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Alice\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-af91-4d1a-aaf2-EXAMPLE\",\"eventID\":\"EXAMPLE-f8e6-4d5f-8525-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "DeleteVirtualMFADevice", + "id": "EXAMPLE-f8e6-4d5f-8525-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "serialNumber": "arn:aws:iam::0123456789012:mfa/Alice" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-09T16:36:17.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{serialNumber=arn:aws:iam::0123456789012:mfa/Alice}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log new file mode 100644 index 000000000000..67cdd3ad6e68 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-11-27T15:07:22Z"}}},"eventTime":"2019-11-27T15:11:09Z","eventSource":"iam.amazonaws.com","eventName":"EnableMFADevice","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"console.amazonaws.com","requestParameters":{"userName":"Bob","serialNumber":"arn:aws:iam::0123456789012:mfa/Bob"},"responseElements":null,"requestID":"EXAMPLE-adea-490a-a806-EXAMPLE","eventID":"EXAMPLE-3fdc-4b2a-9885-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-expected.json new file mode 100644 index 000000000000..d511908596c3 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/enable-mfa-device-json.log-expected.json @@ -0,0 +1,72 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2019-11-27T15:11:09.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.916356700Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-11-27T15:07:22Z\"}}},\"eventTime\":\"2019-11-27T15:11:09Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"EnableMFADevice\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"console.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\",\"serialNumber\":\"arn:aws:iam::0123456789012:mfa/Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-adea-490a-a806-EXAMPLE\",\"eventID\":\"EXAMPLE-3fdc-4b2a-9885-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "EnableMFADevice", + "id": "EXAMPLE-3fdc-4b2a-9885-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "serialNumber": "arn:aws:iam::0123456789012:mfa/Bob" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "session_context": { + "mfa_authenticated": "false", + "creation_date": "2019-11-27T15:07:22.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{serialNumber=arn:aws:iam::0123456789012:mfa/Bob, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "console.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log new file mode 100644 index 000000000000..c5c536fe7a67 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.07","eventTime":"2020-09-09T23:00:00Z","awsRegion":"us-east-1","eventID":"41ed77ca-d659-b45a-8e9a-74e504300007","eventType":"AwsCloudTrailInsight","recipientAccountId":"123456789012","sharedEventID":"e672c2b1-e71a-4779-f96c-02da7bb30d2e","insightDetails":{"state":"End","eventSource":"iam.amazonaws.com","eventName":"AttachUserPolicy","insightType":"ApiCallRateInsight","insffightContext":{"statistics":{"baseline":{"average":0.0},"insight":{"average":2.0},"insightDuration":1,"baselineDuration":11459},"attributions":[{"attribute":"userIdentityArn","insight":[{"value":"arn:aws:iam::123456789012:user/Alice","average":2.0}],"baseline":[]},{"attribute":"userAgent","insight":[{"value":"console.amazonaws.com","average":2.0}],"baseline":[]},{"attribute":"errorCode","insight":[{"value":"null","average":2.0}],"baseline":[]}]}},"eventCategory":"Insight"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-expected.json new file mode 100644 index 000000000000..b3d5089d27de --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/insight-json.log-expected.json @@ -0,0 +1,25 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1" + }, + "@timestamp": "2020-09-09T23:00:00.000Z", + "event": { + "ingested": "2020-11-19T22:16:17.948374300Z", + "original": "{\"eventVersion\":\"1.07\",\"eventTime\":\"2020-09-09T23:00:00Z\",\"awsRegion\":\"us-east-1\",\"eventID\":\"41ed77ca-d659-b45a-8e9a-74e504300007\",\"eventType\":\"AwsCloudTrailInsight\",\"recipientAccountId\":\"123456789012\",\"sharedEventID\":\"e672c2b1-e71a-4779-f96c-02da7bb30d2e\",\"insightDetails\":{\"state\":\"End\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AttachUserPolicy\",\"insightType\":\"ApiCallRateInsight\",\"insffightContext\":{\"statistics\":{\"baseline\":{\"average\":0.0},\"insight\":{\"average\":2.0},\"insightDuration\":1,\"baselineDuration\":11459},\"attributions\":[{\"attribute\":\"userIdentityArn\",\"insight\":[{\"value\":\"arn:aws:iam::123456789012:user/Alice\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"userAgent\",\"insight\":[{\"value\":\"console.amazonaws.com\",\"average\":2.0}],\"baseline\":[]},{\"attribute\":\"errorCode\",\"insight\":[{\"value\":\"null\",\"average\":2.0}],\"baseline\":[]}]}},\"eventCategory\":\"Insight\"}", + "id": "41ed77ca-d659-b45a-8e9a-74e504300007", + "type": "info", + "kind": "event", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.07", + "event_type": "AwsCloudTrailInsight", + "recipient_account_id": "123456789012" + } + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log new file mode 100644 index 000000000000..93c180dfe9b0 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-06T14:36:28Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-06T15:19:50Z","eventSource":"iam.amazonaws.com","eventName":"RemoveUserFromGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"groupName":"Admin","userName":"Bob"},"responseElements":null,"requestID":"EXAMPLE-0bf0-47be-bc80-EXAMPLE","eventID":"EXAMPLE-6e8b-431a-94f4-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-expected.json new file mode 100644 index 000000000000..3a765ccd39d7 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/remove-user-from-group-json.log-expected.json @@ -0,0 +1,73 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-06T15:19:50.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:17.967490300Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-06T14:36:28Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-06T15:19:50Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"RemoveUserFromGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"groupName\":\"Admin\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0bf0-47be-bc80-EXAMPLE\",\"eventID\":\"EXAMPLE-6e8b-431a-94f4-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "RemoveUserFromGroup", + "id": "EXAMPLE-6e8b-431a-94f4-EXAMPLE", + "type": [ + "group", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "groupName": "Admin" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-06T14:36:28.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=Admin, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log new file mode 100644 index 000000000000..e03d924e97bf --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"sessionIssuer":{},"webIdFederationData":{},"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T15:30:25Z","eventSource":"cloudtrail.amazonaws.com","eventName":"StartLogging","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"name":"TEST-trail"},"responseElements":null,"requestID":"EXAMPLE-1c30-4f43-9763-EXAMPLE","eventID":"EXAMPLE-aa78-4a84-a27f-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-expected.json new file mode 100644 index 000000000000..f14857f5d530 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/start-logging-json.log-expected.json @@ -0,0 +1,62 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-west-2", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-08T15:30:25.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.000799400Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T15:30:25Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StartLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-1c30-4f43-9763-EXAMPLE\",\"eventID\":\"EXAMPLE-aa78-4a84-a27f-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "StartLogging", + "id": "EXAMPLE-aa78-4a84-a27f-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "name": "TEST-trail" + } + }, + "event_type": "AwsApiCall", + "read_only": false, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{name=TEST-trail}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log new file mode 100644 index 000000000000..b2c96b814b9d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"sessionIssuer":{},"webIdFederationData":{},"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-09T16:36:17Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-09T16:46:16Z","eventSource":"cloudtrail.amazonaws.com","eventName":"StopLogging","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"name":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail"},"responseElements":null,"requestID":"EXAMPLE-869f-4fec-86f9-EXAMPLE","eventID":"EXAMPLE-8cc3-42db-9a0d-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-expected.json new file mode 100644 index 000000000000..6f833e964877 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/stop-logging-json.log-expected.json @@ -0,0 +1,62 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-west-2", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T16:46:16.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.031847400Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-09T16:36:17Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-09T16:46:16Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"StopLogging\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-869f-4fec-86f9-EXAMPLE\",\"eventID\":\"EXAMPLE-8cc3-42db-9a0d-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "StopLogging", + "id": "EXAMPLE-8cc3-42db-9a0d-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "name": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail" + } + }, + "event_type": "AwsApiCall", + "read_only": false, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-09T16:36:17.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{name=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log new file mode 100644 index 000000000000..ed2b823cfcf2 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T15:01:23Z","eventSource":"iam.amazonaws.com","eventName":"UpdateAccessKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"status":"Inactive","accessKeyId":"EXAMPLE_KEY_ID","userName":"Bob"},"responseElements":null,"requestID":"EXAMPLE-7d0c-45f4-b25b-EXAMPLE","eventID":"EXAMPLE-0ef0-42cd-8551-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-expected.json new file mode 100644 index 000000000000..6bd1a0c7ad10 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-access-key-json.log-expected.json @@ -0,0 +1,74 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T15:01:23.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.065954100Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T15:01:23Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccessKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-7d0c-45f4-b25b-EXAMPLE\",\"eventID\":\"EXAMPLE-0ef0-42cd-8551-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateAccessKey", + "id": "EXAMPLE-0ef0-42cd-8551-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "accessKeyId": "EXAMPLE_KEY_ID", + "userName": "Bob", + "status": "Inactive" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{accessKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Inactive}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log new file mode 100644 index 000000000000..24094717e84d --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T18:05:33Z","eventSource":"iam.amazonaws.com","eventName":"UpdateAccountPasswordPolicy","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"requireLowercaseCharacters":true,"requireSymbols":true,"requireNumbers":true,"minimumPasswordLength":12,"requireUppercaseCharacters":true,"allowUsersToChangePassword":true},"responseElements":null,"requestID":"EXAMPLE-5ebf-4bc3-a349-EXAMPLE","eventID":"EXAMPLE-91f9-49f3-948c-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-expected.json new file mode 100644 index 000000000000..12e1a7a347de --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-accout-password-policy-json.log-expected.json @@ -0,0 +1,72 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T18:05:33.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.099377700Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:05:33Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateAccountPasswordPolicy\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"requireLowercaseCharacters\":true,\"requireSymbols\":true,\"requireNumbers\":true,\"minimumPasswordLength\":12,\"requireUppercaseCharacters\":true,\"allowUsersToChangePassword\":true},\"responseElements\":null,\"requestID\":\"EXAMPLE-5ebf-4bc3-a349-EXAMPLE\",\"eventID\":\"EXAMPLE-91f9-49f3-948c-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateAccountPasswordPolicy", + "id": "EXAMPLE-91f9-49f3-948c-EXAMPLE", + "type": [ + "admin", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "requireLowercaseCharacters": true, + "minimumPasswordLength": 12, + "requireNumbers": true, + "requireSymbols": true, + "requireUppercaseCharacters": true, + "allowUsersToChangePassword": true + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{minimumPasswordLength=12, requireSymbols=true, allowUsersToChangePassword=true, requireLowercaseCharacters=true, requireNumbers=true, requireUppercaseCharacters=true}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log new file mode 100644 index 000000000000..27f9733a7129 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T02:23:11Z","eventSource":"iam.amazonaws.com","eventName":"UpdateGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":{"newGroupName":"TEST-GROUP2","groupName":"TEST-GROUP"},"responseElements":null,"requestID":"EXAMPLE-c22d-4fca-b40a-EXAMPLE","eventID":"EXAMPLE-c3aa-487b-b05e-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T02:24:35Z","eventSource":"iam.amazonaws.com","eventName":"UpdateGroup","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"EntityAlreadyExistsException","errorMessage":"Group with name TEST-GROUP already exists.","requestParameters":{"newGroupName":"TEST-GROUP","groupName":"TEST-GROUP2"},"responseElements":null,"requestID":"EXAMPLE-f673-4ce7-8529-EXAMPLE","eventID":"EXAMPLE-6a0b-475c-b5db-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-expected.json new file mode 100644 index 000000000000..b1f9b9e4844a --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-group-json.log-expected.json @@ -0,0 +1,126 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T02:23:11.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.133366700Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:23:11Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP2\",\"groupName\":\"TEST-GROUP\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-c22d-4fca-b40a-EXAMPLE\",\"eventID\":\"EXAMPLE-c3aa-487b-b05e-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateGroup", + "id": "EXAMPLE-c3aa-487b-b05e-EXAMPLE", + "type": [ + "group", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP", + "newGroupName": "TEST-GROUP2" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP, newGroupName=TEST-GROUP2}" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + }, + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-09T02:24:35.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.133376Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"0123456789012\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-09T02:24:35Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateGroup\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"errorCode\":\"EntityAlreadyExistsException\",\"errorMessage\":\"Group with name TEST-GROUP already exists.\",\"requestParameters\":{\"newGroupName\":\"TEST-GROUP\",\"groupName\":\"TEST-GROUP2\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-f673-4ce7-8529-EXAMPLE\",\"eventID\":\"EXAMPLE-6a0b-475c-b5db-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateGroup", + "id": "EXAMPLE-6a0b-475c-b5db-EXAMPLE", + "type": [ + "group", + "change" + ], + "category": [ + "iam" + ], + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "error_message": "Group with name TEST-GROUP already exists.", + "flattened": { + "request_parameters": { + "groupName": "TEST-GROUP2", + "newGroupName": "TEST-GROUP" + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "error_code": "EntityAlreadyExistsException", + "recipient_account_id": "0123456789012", + "request_parameters": "{groupName=TEST-GROUP2, newGroupName=TEST-GROUP}" + } + }, + "user": { + "name": "Alice", + "id": "0123456789012" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log new file mode 100644 index 000000000000..5dc6e47cb5ec --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T18:25:42Z","eventSource":"iam.amazonaws.com","eventName":"UpdateLoginProfile","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"userName":"Bob"},"responseElements":null,"requestID":"EXAMPLE-0dc6-447a-8859-EXAMPLE","eventID":"EXAMPLE-c3b6-4498-b818-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-expected.json new file mode 100644 index 000000000000..8ff89bf8906b --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-login-profile-json.log-expected.json @@ -0,0 +1,72 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T18:25:42.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.187902Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T18:25:42Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateLoginProfile\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"userName\":\"Bob\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-0dc6-447a-8859-EXAMPLE\",\"eventID\":\"EXAMPLE-c3b6-4498-b818-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateLoginProfile", + "id": "EXAMPLE-c3b6-4498-b818-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log new file mode 100644 index 000000000000..6a31d001b620 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:54Z","eventSource":"iam.amazonaws.com","eventName":"UpdateSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"status":"Inactive","userName":"Bob","sSHPublicKeyId":"EXAMPLE_KEY_ID"},"responseElements":null,"requestID":"EXAMPLE-32f3-4a92-82e1-EXAMPLE","eventID":"EXAMPLE-5c88-4652-9ee9-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:54Z","eventSource":"iam.amazonaws.com","eventName":"UpdateSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"status":"Inactive","userName":"Bob","sSHPublicKeyId":"EXAMPLE_KEY_ID"},"responseElements":null,"requestID":"EXAMPLE-32f3-4a92-82e1-EXAMPLE","eventID":"EXAMPLE-5c88-4652-9ee9-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-expected.json new file mode 100644 index 000000000000..3945eeaf87ac --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-ssh-public-key-json.log-expected.json @@ -0,0 +1,144 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T16:06:54.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.221679200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateSSHPublicKey", + "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "sSHPublicKeyId": "EXAMPLE_KEY_ID", + "status": "Inactive" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{sSHPublicKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Inactive}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + }, + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T16:06:54.000Z", + "related": { + "user": [ + "Bob" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.221688200Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:54Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"status\":\"Inactive\",\"userName\":\"Bob\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\"},\"responseElements\":null,\"requestID\":\"EXAMPLE-32f3-4a92-82e1-EXAMPLE\",\"eventID\":\"EXAMPLE-5c88-4652-9ee9-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateSSHPublicKey", + "id": "EXAMPLE-5c88-4652-9ee9-EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "sSHPublicKeyId": "EXAMPLE_KEY_ID", + "status": "Inactive" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "0123456789012", + "request_parameters": "{sSHPublicKeyId=EXAMPLE_KEY_ID, userName=Bob, status=Inactive}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log new file mode 100644 index 000000000000..f8a9bc9e2a34 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.04","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2016-07-14T19:15:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-east-2","sourceIPAddress":"205.251.233.182","userAgent":"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22","errorCode":"TrailNotFoundException","errorMessage":"Unknown trail: myTrail2 for the user: 123456789012","requestParameters":{"name":"myTrail2"},"responseElements":null,"requestID":"5d40662a-49f7-11e6-97e4-dEXAMPLE","eventID":"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"sessionIssuer":{},"webIdFederationData":{},"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-08T15:12:16Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-08T20:58:45Z","eventSource":"cloudtrail.amazonaws.com","eventName":"UpdateTrail","awsRegion":"us-west-2","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"name":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail","s3BucketName":"test-cloudtrail-bucket","snsTopicName":"","isMultiRegionTrail":true,"enableLogFileValidation":false,"kmsKeyId":""},"responseElements":{"name":"TEST-trail","s3BucketName":"test-cloudtrail-bucket","snsTopicName":"","snsTopicARN":"","includeGlobalServiceEvents":true,"isMultiRegionTrail":true,"trailARN":"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail","logFileValidationEnabled":false,"isOrganizationTrail":false},"requestID":"EXAMPLE-f3da-42d1-84f5-EXAMPLE","eventID":"EXAMPLE-b5e9-4846-8407-EXAMPLE","readOnly":false,"eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-expected.json new file mode 100644 index 000000000000..7797515e826f --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-trail-json.log-expected.json @@ -0,0 +1,155 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-2", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2016-07-14T19:15:45.000Z", + "source": { + "geo": { + "continent_name": "North America", + "region_iso_code": "US-OR", + "city_name": "Boardman", + "country_iso_code": "US", + "country_name": "United States", + "region_name": "Oregon", + "location": { + "lon": -119.7143, + "lat": 45.8491 + } + }, + "as": { + "number": 16509, + "organization": { + "name": "Amazon.com, Inc." + } + }, + "address": "205.251.233.182", + "ip": "205.251.233.182" + }, + "event": { + "ingested": "2020-11-19T22:16:18.282302Z", + "original": "{\"eventVersion\":\"1.04\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2016-07-14T19:15:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"205.251.233.182\",\"userAgent\":\"aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22\",\"errorCode\":\"TrailNotFoundException\",\"errorMessage\":\"Unknown trail: myTrail2 for the user: 123456789012\",\"requestParameters\":{\"name\":\"myTrail2\"},\"responseElements\":null,\"requestID\":\"5d40662a-49f7-11e6-97e4-dEXAMPLE\",\"eventID\":\"b7d4398e-b2f0-4faa-9c76-e2EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "UpdateTrail", + "id": "b7d4398e-b2f0-4faa-9c76-e2EXAMPLE", + "type": "info", + "outcome": "failure" + }, + "aws": { + "cloudtrail": { + "event_version": "1.04", + "error_message": "Unknown trail: myTrail2 for the user: 123456789012", + "flattened": { + "request_parameters": { + "name": "myTrail2" + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "error_code": "TrailNotFoundException", + "recipient_account_id": "123456789012", + "request_parameters": "{name=myTrail2}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.10.32 Python/2.7.9 Windows/7 botocore/1.4.22", + "os": { + "name": "Windows" + }, + "device": { + "name": "Spider" + }, + "version": "1.10.32" + } + }, + { + "cloud": { + "region": "us-west-2", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-08T20:58:45.000Z", + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.282312Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"sessionIssuer\":{},\"webIdFederationData\":{},\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-08T15:12:16Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-08T20:58:45Z\",\"eventSource\":\"cloudtrail.amazonaws.com\",\"eventName\":\"UpdateTrail\",\"awsRegion\":\"us-west-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"name\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"isMultiRegionTrail\":true,\"enableLogFileValidation\":false,\"kmsKeyId\":\"\"},\"responseElements\":{\"name\":\"TEST-trail\",\"s3BucketName\":\"test-cloudtrail-bucket\",\"snsTopicName\":\"\",\"snsTopicARN\":\"\",\"includeGlobalServiceEvents\":true,\"isMultiRegionTrail\":true,\"trailARN\":\"arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail\",\"logFileValidationEnabled\":false,\"isOrganizationTrail\":false},\"requestID\":\"EXAMPLE-f3da-42d1-84f5-EXAMPLE\",\"eventID\":\"EXAMPLE-b5e9-4846-8407-EXAMPLE\",\"readOnly\":false,\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "cloudtrail.amazonaws.com", + "kind": "event", + "action": "UpdateTrail", + "id": "EXAMPLE-b5e9-4846-8407-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "name": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail", + "enableLogFileValidation": false, + "kmsKeyId": "", + "isMultiRegionTrail": true, + "s3BucketName": "test-cloudtrail-bucket", + "snsTopicName": "" + }, + "response_elements": { + "snsTopicARN": "", + "logFileValidationEnabled": false, + "isMultiRegionTrail": true, + "s3BucketName": "test-cloudtrail-bucket", + "snsTopicName": "", + "name": "TEST-trail", + "trailARN": "arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail", + "isOrganizationTrail": false, + "includeGlobalServiceEvents": true + } + }, + "event_type": "AwsApiCall", + "read_only": false, + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-08T15:12:16.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, snsTopicName=, name=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, enableLogFileValidation=false, kmsKeyId=}", + "response_elements": "{snsTopicARN=, logFileValidationEnabled=false, isMultiRegionTrail=true, s3BucketName=test-cloudtrail-bucket, snsTopicName=, name=TEST-trail, trailARN=arn:aws:cloudtrail:us-west-2:0123456789012:trail/TEST-trail, isOrganizationTrail=false, includeGlobalServiceEvents=true}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log new file mode 100644 index 000000000000..62721399a405 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log @@ -0,0 +1,2 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice"},"eventTime":"2020-01-08T20:53:12Z","eventSource":"iam.amazonaws.com","eventName":"UpdateUser","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":{"userName":"Bob","newUserName":"Robert"},"responseElements":null,"requestID":"3a6b3260-739d-465e-9406-bcEXAMPLE","eventID":"9150d546-3564-4262-8e62-110EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"123456789012"} + diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-expected.json new file mode 100644 index 000000000000..3c36d5852116 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/update-user-json.log-expected.json @@ -0,0 +1,78 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "123456789012" + } + }, + "@timestamp": "2020-01-08T20:53:12.000Z", + "related": { + "user": [ + "Bob", + "Robert" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.347266800Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\"},\"eventTime\":\"2020-01-08T20:53:12Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UpdateUser\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46\",\"requestParameters\":{\"userName\":\"Bob\",\"newUserName\":\"Robert\"},\"responseElements\":null,\"requestID\":\"3a6b3260-739d-465e-9406-bcEXAMPLE\",\"eventID\":\"9150d546-3564-4262-8e62-110EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UpdateUser", + "id": "9150d546-3564-4262-8e62-110EXAMPLE", + "type": [ + "user", + "change" + ], + "category": [ + "iam" + ], + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Bob", + "newUserName": "Robert" + } + }, + "user_identity": { + "access_key_id": "EXAMPLE_KEY_ID", + "type": "IAMUser", + "arn": "arn:aws:iam::123456789012:user/Alice" + }, + "event_type": "AwsApiCall", + "recipient_account_id": "123456789012", + "request_parameters": "{newUserName=Robert, userName=Bob}" + } + }, + "user": { + "name": "Alice", + "id": "EX_PRINCIPAL_ID" + }, + "user_agent": { + "name": "aws-cli", + "original": "aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46", + "device": { + "name": "Spider" + }, + "version": "1.16.310" + } + }, + { + "event": { + "ingested": "2020-11-19T22:16:18.347276300Z", + "original": "", + "type": "info", + "kind": "event" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log new file mode 100644 index 000000000000..0db4791855bd --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log @@ -0,0 +1 @@ +{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"EXAMPLE_ID","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"true","creationDate":"2020-01-10T14:38:30Z"}},"invokedBy":"signin.amazonaws.com"},"eventTime":"2020-01-10T16:06:40Z","eventSource":"iam.amazonaws.com","eventName":"UploadSSHPublicKey","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"signin.amazonaws.com","requestParameters":{"sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain","userName":"Alice"},"responseElements":{"sSHPublicKey":{"fingerprint":"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de","status":"Active","uploadDate":"Jan 10, 2020 4:06:40 PM","userName":"Alice","sSHPublicKeyId":"EXAMPLE_KEY_ID","sSHPublicKeyBody":"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain"}},"requestID":"EXAMPLE-44b9-41cd-90f2-EXAMPLE","eventID":"EXAMPLE-9a9d-4da4-9998-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"} diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-config.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-config.json new file mode 100644 index 000000000000..f71947c2f042 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-config.json @@ -0,0 +1,5 @@ +{ + "dynamic_fields": { + "event.ingested": ".*" + } +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-expected.json b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-expected.json new file mode 100644 index 000000000000..6e797c4a2ff2 --- /dev/null +++ b/packages/aws/data_stream/cloudtrail/_dev/test/pipeline/upload-ssh-public-key-json.log-expected.json @@ -0,0 +1,78 @@ +{ + "expected": [ + { + "cloud": { + "region": "us-east-1", + "account": { + "id": "0123456789012" + } + }, + "@timestamp": "2020-01-10T16:06:40.000Z", + "related": { + "user": [ + "Alice" + ] + }, + "source": { + "address": "127.0.0.1", + "ip": "127.0.0.1" + }, + "event": { + "ingested": "2020-11-19T22:16:18.393021500Z", + "original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EXAMPLE_ID\",\"arn\":\"arn:aws:iam::0123456789012:user/Alice\",\"accountId\":\"0123456789012\",\"accessKeyId\":\"EXAMPLE_KEY\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"true\",\"creationDate\":\"2020-01-10T14:38:30Z\"}},\"invokedBy\":\"signin.amazonaws.com\"},\"eventTime\":\"2020-01-10T16:06:40Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"UploadSSHPublicKey\",\"awsRegion\":\"us-east-1\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"signin.amazonaws.com\",\"requestParameters\":{\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\",\"userName\":\"Alice\"},\"responseElements\":{\"sSHPublicKey\":{\"fingerprint\":\"de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de\",\"status\":\"Active\",\"uploadDate\":\"Jan 10, 2020 4:06:40 PM\",\"userName\":\"Alice\",\"sSHPublicKeyId\":\"EXAMPLE_KEY_ID\",\"sSHPublicKeyBody\":\"ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain\"}},\"requestID\":\"EXAMPLE-44b9-41cd-90f2-EXAMPLE\",\"eventID\":\"EXAMPLE-9a9d-4da4-9998-EXAMPLE\",\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"0123456789012\"}", + "provider": "iam.amazonaws.com", + "kind": "event", + "action": "UploadSSHPublicKey", + "id": "EXAMPLE-9a9d-4da4-9998-EXAMPLE", + "type": "info", + "outcome": "success" + }, + "aws": { + "cloudtrail": { + "event_version": "1.05", + "flattened": { + "request_parameters": { + "userName": "Alice", + "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain" + }, + "response_elements": { + "sSHPublicKey": { + "fingerprint": "de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de", + "sSHPublicKeyBody": "ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain", + "sSHPublicKeyId": "EXAMPLE_KEY_ID", + "uploadDate": "Jan 10, 2020 4:06:40 PM", + "userName": "Alice", + "status": "Active" + } + } + }, + "event_type": "AwsApiCall", + "user_identity": { + "access_key_id": "EXAMPLE_KEY", + "invoked_by": "signin.amazonaws.com", + "session_context": { + "mfa_authenticated": "true", + "creation_date": "2020-01-10T14:38:30.000Z" + }, + "type": "IAMUser", + "arn": "arn:aws:iam::0123456789012:user/Alice" + }, + "recipient_account_id": "0123456789012", + "request_parameters": "{sSHPublicKeyBody=ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain, userName=Alice}", + "response_elements": "{sSHPublicKey={sSHPublicKeyBody=ssh-rsa AAAAdeadcodedeadcode Alice@localhost.domain, sSHPublicKeyId=EXAMPLE_KEY_ID, uploadDate=Jan 10, 2020 4:06:40 PM, fingerprint=de:ad:c0:de:de:ad:c0:de:de:ad:c0:de:de:ad:c0:de, userName=Alice, status=Active}}" + } + }, + "user": { + "name": "Alice", + "id": "EXAMPLE_ID" + }, + "user_agent": { + "name": "Other", + "device": { + "name": "Other" + }, + "original": "signin.amazonaws.com" + } + } + ] +} \ No newline at end of file diff --git a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml index e22c986be6f8..033e6387aabd 100644 --- a/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml +++ b/packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml @@ -141,7 +141,7 @@ processors: - rename: field: json.requestParameters target_field: "aws.cloudtrail.flattened.request_parameters" - if: ctx.json.requestParameters != null + if: ctx.json?.requestParameters != null - script: lang: painless source: | @@ -152,7 +152,7 @@ processors: - rename: field: json.responseElements target_field: "aws.cloudtrail.flattened.response_elements" - if: ctx.json.responseElements != null + if: ctx.json?.responseElements != null - script: lang: painless source: | @@ -214,7 +214,7 @@ processors: - rename: field: json.serviceEventDetails target_field: "aws.cloudtrail.flattened.service_event_details" - if: ctx.json.serviceEventDetails != null + if: ctx.json?.serviceEventDetails != null - script: lang: painless source: | diff --git a/packages/aws/data_stream/cloudtrail/fields/ecs.yml b/packages/aws/data_stream/cloudtrail/fields/ecs.yml index bc61ccdfe6a5..78c350fbd12e 100644 --- a/packages/aws/data_stream/cloudtrail/fields/ecs.yml +++ b/packages/aws/data_stream/cloudtrail/fields/ecs.yml @@ -1,3 +1,6 @@ +- name: error.message + type: text + description: Error message. - name: event.action type: keyword description: The action captured by the event. @@ -34,6 +37,18 @@ - name: user_agent.original type: keyword description: Unparsed user_agent string. +- name: user_agent.os.full + type: keyword + description: Operating system name, including the version or code name. +- name: user_agent.os.name + type: keyword + description: Operating system name, without the version. +- name: user_agent.os.version + type: keyword + description: Operating system version as a raw string. +- name: user_agent.version + type: keyword + description: Version of the user agent. - name: related.user type: keyword description: All the user names seen on your event. diff --git a/packages/aws/data_stream/cloudtrail/fields/fields.yml b/packages/aws/data_stream/cloudtrail/fields/fields.yml index 26d576c1dc8c..5a12696f9f2d 100644 --- a/packages/aws/data_stream/cloudtrail/fields/fields.yml +++ b/packages/aws/data_stream/cloudtrail/fields/fields.yml @@ -89,7 +89,7 @@ type: keyword description: A Boolean value that identifies whether the event is a management event. - name: read_only - type: keyword + type: boolean description: Identifies whether this operation is a read-only operation. - name: resources type: group diff --git a/packages/aws/manifest.yml b/packages/aws/manifest.yml index beaef6785938..21e870953c7e 100644 --- a/packages/aws/manifest.yml +++ b/packages/aws/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: aws title: AWS -version: 0.3.12 +version: 0.3.13 license: basic description: AWS Integration type: integration