Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Filebeat] Check types in googlecloud/audit field renames #20465

Merged
merged 2 commits into from
Aug 6, 2020
Merged

[Filebeat] Check types in googlecloud/audit field renames #20465

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9a7bfce
Check types in googlecloud/audit field renames
andrewkroh Aug 6, 2020
f9e1ed9
Add test event without authorization_info[]
andrewkroh Aug 6, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
- Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
- Fix `okta` geoip lookup in pipeline for `destination.ip` {pull}20454[20454]
- Fix mapping exception in the `googlecloud/audit` dataset pipeline. {issue}18465[18465] {pull}20465[20465]

*Heartbeat*

Expand Down
257 changes: 192 additions & 65 deletions x-pack/filebeat/module/googlecloud/audit/config/pipeline.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,24 @@ function Audit(keep_original_message) {
ignore_missing: true
});

// Use the monitored resource type's labels to set the cloud metadata.
// The labels can vary based on the resource.type.
// https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource
var setCloudMetadata = new processor.Convert({
fields: [
{from: "json.resource.labels.project_id", to: "cloud.project.id"},
{
from: "json.resource.labels.project_id",
to: "cloud.project.id",
type: "string"
},
{
from: "json.resource.labels.instance_id",
to: "cloud.instance.id",
type: "string"
}
],
ignore_missing: true
ignore_missing: true,
fail_on_error: false,
});

// The log includes a protoPayload field.
Expand All @@ -61,99 +74,213 @@ function Audit(keep_original_message) {

// The LogEntry's protoPayload is moved to the json field. The protoPayload
// contains the structured audit log fields.
// https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog
var convertProtoPayload = new processor.Convert({
fields: [
{from: "json.@type", to: "googlecloud.audit.type"},

{from: "json.authenticationInfo.principalEmail", to: "json.authenticationInfo.principal_email"},
{from: "json.authenticationInfo.authoritySelector", to: "json.authenticationInfo.authority_selector"},
{from: "json.authenticationInfo", to: "googlecloud.audit.authentication_info"},

{from: "json.authorizationInfo", to: "googlecloud.audit.authorization_info"},

{from: "json.methodName", to: "googlecloud.audit.method_name"},

{from: "json.numResponseItems", to: "googlecloud.audit.num_response_items", type: "long"},

{from: "json.request.@type", to: "googlecloud.audit.request.proto_name"},
{from: "json.request.filter", to: "googlecloud.audit.request.filter"},
{from: "json.request.name", to: "googlecloud.audit.request.name"},
{from: "json.request.resourceName", to: "googlecloud.audit.request.resource_name"},

{from: "json.requestMetadata.callerIp", to: "json.requestMetadata.caller_ip", type: "ip"},
{from: "json.requestMetadata.callerSuppliedUserAgent", to: "json.requestMetadata.caller_supplied_user_agent"},
{from: "json.requestMetadata", to: "googlecloud.audit.request_metadata"},

{from: "json.response.@type", to: "googlecloud.audit.response.proto_name"},
{from: "json.response.status", to: "googlecloud.audit.response.status"},
{from: "json.response.details.group", to: "googlecloud.audit.response.details.group"},
{from: "json.response.details.kind", to: "googlecloud.audit.response.details.kind"},
{from: "json.response.details.name", to: "googlecloud.audit.response.details.name"},
{from: "json.response.details.uid", to: "googlecloud.audit.response.details.uid"},

{from: "json.resourceName", to: "googlecloud.audit.resource_name"},

{from: "json.resourceLocation.currentLocations", to: "json.resourceLocation.current_locations"},
{from: "json.resourceLocation", to: "googlecloud.audit.resource_location"},

{from: "json.serviceName", to: "googlecloud.audit.service_name"},

{from: "json.status", to: "googlecloud.audit.status"},

{
from: "json.@type",
to: "googlecloud.audit.type",
type: "string"
},
{
from: "json.authenticationInfo.principalEmail",
to: "googlecloud.audit.authentication_info.principal_email",
type: "string"
},
{
from: "json.authenticationInfo.authoritySelector",
to: "googlecloud.audit.authentication_info.authority_selector",
type: "string"
},
{
from: "json.authorizationInfo",
to: "googlecloud.audit.authorization_info"
// Type is an array of objects.
},
{
from: "json.methodName",
to: "googlecloud.audit.method_name",
type: "string",
},
{
from: "json.numResponseItems",
to: "googlecloud.audit.num_response_items",
type: "long"
},
{
from: "json.request.@type",
to: "googlecloud.audit.request.proto_name",
type: "string"
},
// The values in the request object will depend on the proto type.
// So be very careful about making any assumptions about data shape.
{
from: "json.request.filter",
to: "googlecloud.audit.request.filter",
type: "string"
},
{
from: "json.request.name",
to: "googlecloud.audit.request.name",
type: "string"
},
{
from: "json.request.resourceName",
to: "googlecloud.audit.request.resource_name",
type: "string"
},
{
from: "json.requestMetadata.callerIp",
to: "googlecloud.audit.request_metadata.caller_ip",
type: "ip"
},
{
from: "json.requestMetadata.callerSuppliedUserAgent",
to: "googlecloud.audit.request_metadata.caller_supplied_user_agent",
type: "string",
},
{
from: "json.response.@type",
to: "googlecloud.audit.response.proto_name",
type: "string"
},
// The values in the response object will depend on the proto type.
// So be very careful about making any assumptions about data shape.
{
from: "json.response.status",
to: "googlecloud.audit.response.status",
type: "string"
},
{
from: "json.response.details.group",
to: "googlecloud.audit.response.details.group",
type: "string"
},
{
from: "json.response.details.kind",
to: "googlecloud.audit.response.details.kind",
type: "string"
},
{
from: "json.response.details.name",
to: "googlecloud.audit.response.details.name",
type: "string"
},
{
from: "json.response.details.uid",
to: "googlecloud.audit.response.details.uid",
type: "string",
},
{
from: "json.resourceName",
to: "googlecloud.audit.resource_name",
type: "string",
},
{
from: "json.resourceLocation.currentLocations",
to: "googlecloud.audit.resource_location.current_locations"
// Type is a string array.
},
{
from: "json.serviceName",
to: "googlecloud.audit.service_name",
type: "string",
},
{
from: "json.status.code",
to: "googlecloud.audit.status.code",
type: "integer",
},
{
from: "json.status.message",
to: "googlecloud.audit.status.message",
type: "string"
},
],
mode: "rename",
ignore_missing: true,
fail_on_error: false,
});

// Copy some fields
var copyFields = new processor.Convert({
fields: [
{from: "googlecloud.audit.request_metadata.caller_ip", to: "source.ip"},
{from: "googlecloud.audit.authentication_info.principal_email", to: "user.email"},
{from: "googlecloud.audit.service_name", to: "service.name"},
{from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", to: "user_agent.original"},
{from: "googlecloud.audit.method_name", to: "event.action"},
{
from: "googlecloud.audit.request_metadata.caller_ip",
to: "source.ip",
type: "ip"
},
{
from: "googlecloud.audit.authentication_info.principal_email",
to: "user.email",
type: "string"
},
{
from: "googlecloud.audit.service_name",
to: "service.name",
type: "string"
},
{
from: "googlecloud.audit.request_metadata.caller_supplied_user_agent",
to: "user_agent.original",
type: "string"
},
{
from: "googlecloud.audit.method_name",
to: "event.action",
type: "string"
},
],
ignore_missing: true,
fail_on_error: false,
});

// Drop extra fields
var dropExtraFields = function(evt) {
evt.Delete("json");
evt.Delete("googlecloud.audit.request_metadata.requestAttributes");
evt.Delete("googlecloud.audit.request_metadata.destinationAttributes");
};

// Rename nested fields
var RenameNestedFields = function(evt) {
// Rename nested fields.
var renameNestedFields = function(evt) {
var arr = evt.Get("googlecloud.audit.authorization_info");
for (var i = 0; i < arr.length; i++) {
arr[i].resource_attributes = arr[i].resourceAttributes;
delete arr[i].resourceAttributes;
if (Array.isArray(arr)) {
for (var i = 0; i < arr.length; i++) {
if (arr[i].resourceAttributes) {
// Convert to snake_case.
arr[i].resource_attributes = arr[i].resourceAttributes;
delete arr[i].resourceAttributes;
}
}
}
};

// Set ECS categorization fields.
var setECSCategorization = function(evt) {
evt.Put("event.kind", "event");

// google.rpc.Code value for OK is 0.
if (evt.Get("googlecloud.audit.status.code") === 0) {
evt.Put("event.outcome", "success");
return;
}

// Try to use authorization_info.granted when there was no status code.
if (evt.Get("googlecloud.audit.status.code") == null) {
var authorization_info = evt.Get("googlecloud.audit.authorization_info");
if (authorization_info.length === 1) {
if (authorization_info[0].granted == null) {
evt.Put("event.outcome", "unknown");
} else if (authorization_info[0].granted === true) {
if (Array.isArray(authorization_info) && authorization_info.length === 1) {
if (authorization_info[0].granted === true) {
evt.Put("event.outcome", "success");
} else {
} else if (authorization_info[0].granted === false) {
evt.Put("event.outcome", "failure");
}
} else {
evt.Put("event.outcome", "unknown");
}
} else if (evt.Get("googlecloud.audit.status.code") === 0) {
evt.Put("event.outcome", "success");
} else {
evt.Put("event.outcome", "failure");
return
}

evt.Put("event.outcome", "unknown");
return;
}
evt.Put("event.kind", "event");

evt.Put("event.outcome", "failure");
};

var pipeline = new processor.Chain()
Expand All @@ -167,7 +294,7 @@ function Audit(keep_original_message) {
.Add(convertProtoPayload)
.Add(copyFields)
.Add(dropExtraFields)
.Add(RenameNestedFields)
.Add(renameNestedFields)
.Add(setECSCategorization)
.Build();

Expand Down
Loading