Skip to content

Commit

Permalink
Add pipeline tests for AWS CloudTrail
Browse files Browse the repository at this point in the history 
The tests revealed a few issues. There was an error in the pipeline for update-user-json.log because
serviceEventDetails was not present. This was the error

            "error": {
                "message": "Cannot invoke \\\"Object.getClass()\\\" because \\\"receiver\\\" is null"
            }

The aws.cloudtrail.read_only field was mapped as keyword but was actual a JSON boolean.
I changed the type to boolean, but do not plan to backport this change to Filebeat.

And lastly some ECS user_agent fields were missing.

This depends on elastic/elastic-package#177 to make the flattened fields pass
test validation.
  • Loading branch information
@andrewkroh
andrewkroh committed Dec 1, 2020
1 parent 3bf1cd7 commit e13a28f
Show file tree
Hide file tree
Showing 101 changed files with 3,032 additions and 6 deletions.
1 change: 1 addition & 0 deletions packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"eventVersion":"1.0","userIdentity":{"type":"IAMUser","principalId":"EX_PRINCIPAL_ID","arn":"arn:aws:iam::123456789012:user/Alice","accountId":"123456789012","accessKeyId":"EXAMPLE_KEY_ID","userName":"Alice","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2014-03-25T18:45:11Z"}}},"eventTime":"2014-03-25T21:08:14Z","eventSource":"iam.amazonaws.com","eventName":"AddUserToGroup","awsRegion":"us-east-2","sourceIPAddress":"127.0.0.1","userAgent":"AWSConsole","requestParameters":{"userName":"Bob","groupName":"admin"},"responseElements":null}
5 changes: 5 additions & 0 deletions ...aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-config.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"dynamic_fields": {
"event.ingested": ".*"
}
}
69 changes: 69 additions & 0 deletions ...s/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
{
"expected": [
{
"cloud": {
"region": "us-east-2",
"account": {
"id": "123456789012"
}
},
"@timestamp": "2014-03-25T21:08:14.000Z",
"related": {
"user": [
"Bob"
]
},
"source": {
"address": "127.0.0.1",
"ip": "127.0.0.1"
},
"event": {
"ingested": "2020-11-19T22:16:17.114840Z",
"original": "{\"eventVersion\":\"1.0\",\"userIdentity\":{\"type\":\"IAMUser\",\"principalId\":\"EX_PRINCIPAL_ID\",\"arn\":\"arn:aws:iam::123456789012:user/Alice\",\"accountId\":\"123456789012\",\"accessKeyId\":\"EXAMPLE_KEY_ID\",\"userName\":\"Alice\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2014-03-25T18:45:11Z\"}}},\"eventTime\":\"2014-03-25T21:08:14Z\",\"eventSource\":\"iam.amazonaws.com\",\"eventName\":\"AddUserToGroup\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"127.0.0.1\",\"userAgent\":\"AWSConsole\",\"requestParameters\":{\"userName\":\"Bob\",\"groupName\":\"admin\"},\"responseElements\":null}",
"provider": "iam.amazonaws.com",
"kind": "event",
"action": "AddUserToGroup",
"type": [
"group",
"change"
],
"category": [
"iam"
],
"outcome": "success"
},
"aws": {
"cloudtrail": {
"event_version": "1.0",
"flattened": {
"request_parameters": {
"userName": "Bob",
"groupName": "admin"
}
},
"user_identity": {
"access_key_id": "EXAMPLE_KEY_ID",
"session_context": {
"mfa_authenticated": "false",
"creation_date": "2014-03-25T18:45:11.000Z"
},
"type": "IAMUser",
"arn": "arn:aws:iam::123456789012:user/Alice"
},
"request_parameters": "{groupName=admin, userName=Bob}"
}
},
"user": {
"name": "Alice",
"id": "EX_PRINCIPAL_ID"
},
"user_agent": {
"name": "Other",
"device": {
"name": "Other"
},
"original": "AWSConsole"
}
}
]
}
1 change: 1 addition & 0 deletions packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"eventVersion":"1.05","userIdentity":{"type":"AssumedRole","principalId":"AROAIN5ATK5U7KEXAMPLE:JohnRole1","arn":"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1","accountId":"111111111111","accessKeyId":"AKIAI44QH8DHBEXAMPLE","sessionContext":{"attributes":{"mfaAuthenticated":"false","creationDate":"2019-10-02T21:50:54Z"},"sessionIssuer":{"type":"Role","principalId":"AROAIN5ATK5U7KEXAMPLE","arn":"arn:aws:iam::111111111111:role/JohnRole1","accountId":"111111111111","userName":"JohnDoe"}}},"eventTime":"2019-10-02T22:12:29Z","eventSource":"sts.amazonaws.com","eventName":"AssumeRole","awsRegion":"us-east-2","sourceIPAddress":"123.145.67.89","userAgent":"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239","requestParameters":{"incomingTransitiveTags":{"Department":"Engineering"},"tags":[{"value":"johndoe@example.com","key":"Email"},{"value":"12345","key":"CostCenter"}],"roleArn":"arn:aws:iam::111111111111:role/JohnRole2","roleSessionName":"Role2WithTags","transitiveTagKeys":["Email","CostCenter"],"durationSeconds":3600},"responseElements":{"credentials":{"accessKeyId":"ASIAWHOJDLGPOEXAMPLE","expiration":"Oct 2, 2019 11:12:29 PM","sessionToken":"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN"},"assumedRoleUser":{"assumedRoleId":"AROAIFR7WHDTSOYQYHFUE:Role2WithTags","arn":"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"}},"requestID":"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE","eventID":"1917948f-3042-46ec-98e2-62865EXAMPLE","resources":[{"ARN":"arn:aws:iam::111122223333:role/JohnRole2","accountId":"111111111111","type":"AWS::IAM::Role"}],"eventType":"AwsApiCall","recipientAccountId":"111111111111"}
5 changes: 5 additions & 0 deletions packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-config.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"dynamic_fields": {
"event.ingested": ".*"
}
}
125 changes: 125 additions & 0 deletions ...ges/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
{
"expected": [
{
"cloud": {
"region": "us-east-2",
"account": {
"id": "111111111111"
}
},
"@timestamp": "2019-10-02T22:12:29.000Z",
"source": {
"geo": {
"continent_name": "Asia",
"region_iso_code": "CN-CQ",
"country_name": "China",
"region_name": "Chongqing",
"location": {
"lon": 106.5531,
"lat": 29.5569
},
"country_iso_code": "CN"
},
"as": {
"number": 4837,
"organization": {
"name": "CHINA UNICOM China169 Backbone"
}
},
"address": "123.145.67.89",
"ip": "123.145.67.89"
},
"event": {
"ingested": "2020-11-19T22:16:17.142969600Z",
"original": "{\"eventVersion\":\"1.05\",\"userIdentity\":{\"type\":\"AssumedRole\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE:JohnRole1\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1\",\"accountId\":\"111111111111\",\"accessKeyId\":\"AKIAI44QH8DHBEXAMPLE\",\"sessionContext\":{\"attributes\":{\"mfaAuthenticated\":\"false\",\"creationDate\":\"2019-10-02T21:50:54Z\"},\"sessionIssuer\":{\"type\":\"Role\",\"principalId\":\"AROAIN5ATK5U7KEXAMPLE\",\"arn\":\"arn:aws:iam::111111111111:role/JohnRole1\",\"accountId\":\"111111111111\",\"userName\":\"JohnDoe\"}}},\"eventTime\":\"2019-10-02T22:12:29Z\",\"eventSource\":\"sts.amazonaws.com\",\"eventName\":\"AssumeRole\",\"awsRegion\":\"us-east-2\",\"sourceIPAddress\":\"123.145.67.89\",\"userAgent\":\"aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239\",\"requestParameters\":{\"incomingTransitiveTags\":{\"Department\":\"Engineering\"},\"tags\":[{\"value\":\"johndoe@example.com\",\"key\":\"Email\"},{\"value\":\"12345\",\"key\":\"CostCenter\"}],\"roleArn\":\"arn:aws:iam::111111111111:role/JohnRole2\",\"roleSessionName\":\"Role2WithTags\",\"transitiveTagKeys\":[\"Email\",\"CostCenter\"],\"durationSeconds\":3600},\"responseElements\":{\"credentials\":{\"accessKeyId\":\"ASIAWHOJDLGPOEXAMPLE\",\"expiration\":\"Oct 2, 2019 11:12:29 PM\",\"sessionToken\":\"AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN\"},\"assumedRoleUser\":{\"assumedRoleId\":\"AROAIFR7WHDTSOYQYHFUE:Role2WithTags\",\"arn\":\"arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags\"}},\"requestID\":\"b96b0e4e-e561-11e9-8b3f-7b396EXAMPLE\",\"eventID\":\"1917948f-3042-46ec-98e2-62865EXAMPLE\",\"resources\":[{\"ARN\":\"arn:aws:iam::111122223333:role/JohnRole2\",\"accountId\":\"111111111111\",\"type\":\"AWS::IAM::Role\"}],\"eventType\":\"AwsApiCall\",\"recipientAccountId\":\"111111111111\"}",
"provider": "sts.amazonaws.com",
"kind": "event",
"action": "AssumeRole",
"id": "1917948f-3042-46ec-98e2-62865EXAMPLE",
"type": [
"info"
],
"category": [
"authentication"
],
"outcome": "success"
},
"aws": {
"cloudtrail": {
"event_version": "1.05",
"flattened": {
"request_parameters": {
"incomingTransitiveTags": {
"Department": "Engineering"
},
"transitiveTagKeys": [
"Email",
"CostCenter"
],
"durationSeconds": 3600,
"roleArn": "arn:aws:iam::111111111111:role/JohnRole2",
"roleSessionName": "Role2WithTags",
"tags": [
{
"value": "johndoe@example.com",
"key": "Email"
},
{
"value": "12345",
"key": "CostCenter"
}
]
},
"response_elements": {
"assumedRoleUser": {
"assumedRoleId": "AROAIFR7WHDTSOYQYHFUE:Role2WithTags",
"arn": "arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags"
},
"credentials": {
"accessKeyId": "ASIAWHOJDLGPOEXAMPLE",
"sessionToken": "AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN",
"expiration": "Oct 2, 2019 11:12:29 PM"
}
}
},
"event_type": "AwsApiCall",
"user_identity": {
"access_key_id": "AKIAI44QH8DHBEXAMPLE",
"session_context": {
"mfa_authenticated": "false",
"session_issuer": {
"account_id": "111111111111",
"type": "Role",
"arn": "arn:aws:iam::111111111111:role/JohnRole1",
"principal_id": "AROAIN5ATK5U7KEXAMPLE"
},
"creation_date": "2019-10-02T21:50:54.000Z"
},
"type": "AssumedRole",
"arn": "arn:aws:sts::111111111111:assumed-role/JohnDoe/JohnRole1"
},
"recipient_account_id": "111111111111",
"request_parameters": "{incomingTransitiveTags={Department=Engineering}, transitiveTagKeys=[Email, CostCenter], durationSeconds=3600, roleArn=arn:aws:iam::111111111111:role/JohnRole2, roleSessionName=Role2WithTags, tags=[{value=johndoe@example.com, key=Email}, {value=12345, key=CostCenter}]}",
"response_elements": "{assumedRoleUser={assumedRoleId=AROAIFR7WHDTSOYQYHFUE:Role2WithTags, arn=arn:aws:sts::111111111111:assumed-role/test-role/Role2WithTags}, credentials={accessKeyId=ASIAWHOJDLGPOEXAMPLE, sessionToken=AgoJb3JpZ2luX2VjEB4aCXVzLXdlc3QtMSJHMEXAMPLETOKEN+//rJb8Lo30mFc5MlhFCEbubZvEj0wHB/mDMwIgSEe9gk/Zjr09tZV7F1HDTMhmEXAMPLETOKEN/iEJ/rkqngII9///////////ARABGgw0MjgzMDc4NjM5NjYiDLZjZFKwP4qxQG5sFCryASO4UPz5qE97wPPH1eLMvs7CgSDBSWfonmRTCfokm2FN1+hWUdQQH6adjbbrVLFL8c3jSsBhQ383AvxpwK5YRuDE1AI/+C+WKFZb701eiv9J5La2EXAMPLETOKEN/c7S5Iro1WUJ0q3Cxuo/8HUoSxVhQHM7zF7mWWLhXLEQ52ivL+F6q5dpXu4aTFedpMfnJa8JtkWwG9x1Axj0Ypy2ok8v5unpQGWych1vwdvj6ez1Dm8Xg1+qIzXILiEXAMPLETOKEN/vQGqu8H+nxp3kabcrtOvTFTvxX6vsc8OGwUfHhzAfYGEXAMPLETOKEN/L6v1yMM3B1OwFOrQBno1HEjf1oNI8RnQiMNFdUOtwYj7HUZIOCZmjfN8PPHq77N7GJl9lzvIZKQA0Owcjg+mc78zHCj8y0siY8C96paEXAMPLETOKEN/E3cpksxWdgs91HRzJWScjN2+r2LTGjYhyPqcmFzzo2mCE7mBNEXAMPLETOKEN/oJy+2o83YNW5tOiDmczgDzJZ4UKR84yGYOMfSnF4XcEJrDgAJ3OJFwmTcTQICAlSwLEXAMPLETOKEN, expiration=Oct 2, 2019 11:12:29 PM}}"
}
},
"user": {
"name": "JohnDoe",
"id": "AROAIN5ATK5U7KEXAMPLE:JohnRole1"
},
"user_agent": {
"name": "aws-cli",
"original": "aws-cli/1.16.248 Python/3.4.7 Linux/4.9.184-0.1.ac.235.83.329.metal1.x86_64 botocore/1.12.239",
"os": {
"name": "Linux",
"version": "4.9.184",
"full": "Linux 4.9.184"
},
"device": {
"name": "Spider"
},
"version": "1.16.248"
}
}
]
}
2 changes: 2 additions & 0 deletions packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:09:33Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","errorCode":"AccessDeniedException","errorMessage":"An unknown error occurred","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5204-4fed-9c60-9c6EXAMPLE","eventID":"EXAMPLE-b92f-48bb-8c4c-efeEXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}
{"eventVersion":"1.05","userIdentity":{"type":"IAMUser","principalId":"0123456789012","arn":"arn:aws:iam::0123456789012:user/Alice","accountId":"0123456789012","accessKeyId":"EXAMPLE_KEY","userName":"Alice"},"eventTime":"2020-01-09T00:03:36Z","eventSource":"iam.amazonaws.com","eventName":"ChangePassword","awsRegion":"us-east-1","sourceIPAddress":"127.0.0.1","userAgent":"aws-cli/1.16.310 Python/3.8.1 Darwin/18.7.0 botocore/1.13.46","requestParameters":null,"responseElements":null,"requestID":"EXAMPLE-5c16-4eda-9724-EXAMPLE","eventID":"EXAMPLE-35a7-4c25-9fc7-EXAMPLE","eventType":"AwsApiCall","recipientAccountId":"0123456789012"}
5 changes: 5 additions & 0 deletions ...s/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-config.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"dynamic_fields": {
"event.ingested": ".*"
}
}
Loading

0 comments on commit e13a28f

Please sign in to comment.