From 9a7bfce85f623dbdf9e83b509a66bc890313a6ec Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 5 Aug 2020 23:01:49 -0400 Subject: [PATCH 1/2] Check types in googlecloud/audit field renames Specify type to the convert processor so that fields are renamed only if the value is (or can be coerced to) the desired type. This prevents the module from writing incompatible data types into fields and prevents mapping exceptions. The data in the google cloud audit logs can vary based on the source service so any time the module makes an assumption about a field that's not explicitly documented the module should be very defensive. The request/response object values would be good candidates for the flattened data type if we want to make a future change. Fixes #18465 --- .../googlecloud/audit/config/pipeline.js | 240 +++++++++++++----- .../audit/test/audit-log-entries.json.log | 2 + .../audit-log-entries.json.log-expected.json | 102 ++++++++ 3 files changed, 287 insertions(+), 57 deletions(-) diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js index c467cc77c87..dbfa0479a93 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js +++ b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js @@ -43,11 +43,24 @@ function Audit(keep_original_message) { ignore_missing: true }); + // Use the monitored resource type's labels to set the cloud metadata. + // The labels can vary based on the resource.type. + // https://cloud.google.com/logging/docs/reference/v2/rest/v2/MonitoredResource var setCloudMetadata = new processor.Convert({ fields: [ - {from: "json.resource.labels.project_id", to: "cloud.project.id"}, + { + from: "json.resource.labels.project_id", + to: "cloud.project.id", + type: "string" + }, + { + from: "json.resource.labels.instance_id", + to: "cloud.instance.id", + type: "string" + } ], - ignore_missing: true + ignore_missing: true, + fail_on_error: false, }); // The log includes a protoPayload field. @@ -61,80 +74,195 @@ function Audit(keep_original_message) { // The LogEntry's protoPayload is moved to the json field. The protoPayload // contains the structured audit log fields. + // https://cloud.google.com/logging/docs/reference/audit/auditlog/rest/Shared.Types/AuditLog var convertProtoPayload = new processor.Convert({ fields: [ - {from: "json.@type", to: "googlecloud.audit.type"}, - - {from: "json.authenticationInfo.principalEmail", to: "json.authenticationInfo.principal_email"}, - {from: "json.authenticationInfo.authoritySelector", to: "json.authenticationInfo.authority_selector"}, - {from: "json.authenticationInfo", to: "googlecloud.audit.authentication_info"}, - - {from: "json.authorizationInfo", to: "googlecloud.audit.authorization_info"}, - - {from: "json.methodName", to: "googlecloud.audit.method_name"}, - - {from: "json.numResponseItems", to: "googlecloud.audit.num_response_items", type: "long"}, - - {from: "json.request.@type", to: "googlecloud.audit.request.proto_name"}, - {from: "json.request.filter", to: "googlecloud.audit.request.filter"}, - {from: "json.request.name", to: "googlecloud.audit.request.name"}, - {from: "json.request.resourceName", to: "googlecloud.audit.request.resource_name"}, - - {from: "json.requestMetadata.callerIp", to: "json.requestMetadata.caller_ip", type: "ip"}, - {from: "json.requestMetadata.callerSuppliedUserAgent", to: "json.requestMetadata.caller_supplied_user_agent"}, - {from: "json.requestMetadata", to: "googlecloud.audit.request_metadata"}, - - {from: "json.response.@type", to: "googlecloud.audit.response.proto_name"}, - {from: "json.response.status", to: "googlecloud.audit.response.status"}, - {from: "json.response.details.group", to: "googlecloud.audit.response.details.group"}, - {from: "json.response.details.kind", to: "googlecloud.audit.response.details.kind"}, - {from: "json.response.details.name", to: "googlecloud.audit.response.details.name"}, - {from: "json.response.details.uid", to: "googlecloud.audit.response.details.uid"}, - - {from: "json.resourceName", to: "googlecloud.audit.resource_name"}, - - {from: "json.resourceLocation.currentLocations", to: "json.resourceLocation.current_locations"}, - {from: "json.resourceLocation", to: "googlecloud.audit.resource_location"}, - - {from: "json.serviceName", to: "googlecloud.audit.service_name"}, - - {from: "json.status", to: "googlecloud.audit.status"}, - + { + from: "json.@type", + to: "googlecloud.audit.type", + type: "string" + }, + { + from: "json.authenticationInfo.principalEmail", + to: "googlecloud.audit.authentication_info.principal_email", + type: "string" + }, + { + from: "json.authenticationInfo.authoritySelector", + to: "googlecloud.audit.authentication_info.authority_selector", + type: "string" + }, + { + from: "json.authorizationInfo", + to: "googlecloud.audit.authorization_info" + // Type is an array of objects. + }, + { + from: "json.methodName", + to: "googlecloud.audit.method_name", + type: "string", + }, + { + from: "json.numResponseItems", + to: "googlecloud.audit.num_response_items", + type: "long" + }, + { + from: "json.request.@type", + to: "googlecloud.audit.request.proto_name", + type: "string" + }, + // The values in the request object will depend on the proto type. + // So be very careful about making any assumptions about data shape. + { + from: "json.request.filter", + to: "googlecloud.audit.request.filter", + type: "string" + }, + { + from: "json.request.name", + to: "googlecloud.audit.request.name", + type: "string" + }, + { + from: "json.request.resourceName", + to: "googlecloud.audit.request.resource_name", + type: "string" + }, + { + from: "json.requestMetadata.callerIp", + to: "googlecloud.audit.request_metadata.caller_ip", + type: "ip" + }, + { + from: "json.requestMetadata.callerSuppliedUserAgent", + to: "googlecloud.audit.request_metadata.caller_supplied_user_agent", + type: "string", + }, + { + from: "json.response.@type", + to: "googlecloud.audit.response.proto_name", + type: "string" + }, + // The values in the response object will depend on the proto type. + // So be very careful about making any assumptions about data shape. + { + from: "json.response.status", + to: "googlecloud.audit.response.status", + type: "string" + }, + { + from: "json.response.details.group", + to: "googlecloud.audit.response.details.group", + type: "string" + }, + { + from: "json.response.details.kind", + to: "googlecloud.audit.response.details.kind", + type: "string" + }, + { + from: "json.response.details.name", + to: "googlecloud.audit.response.details.name", + type: "string" + }, + { + from: "json.response.details.uid", + to: "googlecloud.audit.response.details.uid", + type: "string", + }, + { + from: "json.resourceName", + to: "googlecloud.audit.resource_name", + type: "string", + }, + { + from: "json.resourceLocation.currentLocations", + to: "googlecloud.audit.resource_location.current_locations" + // Type is a string array. + }, + { + from: "json.serviceName", + to: "googlecloud.audit.service_name", + type: "string", + }, + { + from: "json.status.code", + to: "googlecloud.audit.status.code", + type: "integer", + }, + { + from: "json.status.message", + to: "googlecloud.audit.status.message", + type: "string" + }, ], mode: "rename", ignore_missing: true, + fail_on_error: false, }); // Copy some fields var copyFields = new processor.Convert({ fields: [ - {from: "googlecloud.audit.request_metadata.caller_ip", to: "source.ip"}, - {from: "googlecloud.audit.authentication_info.principal_email", to: "user.email"}, - {from: "googlecloud.audit.service_name", to: "service.name"}, - {from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", to: "user_agent.original"}, - {from: "googlecloud.audit.method_name", to: "event.action"}, + { + from: "googlecloud.audit.request_metadata.caller_ip", + to: "source.ip", + type: "ip" + }, + { + from: "googlecloud.audit.authentication_info.principal_email", + to: "user.email", + type: "string" + }, + { + from: "googlecloud.audit.service_name", + to: "service.name", + type: "string" + }, + { + from: "googlecloud.audit.request_metadata.caller_supplied_user_agent", + to: "user_agent.original", + type: "string" + }, + { + from: "googlecloud.audit.method_name", + to: "event.action", + type: "string" + }, ], + ignore_missing: true, fail_on_error: false, }); // Drop extra fields var dropExtraFields = function(evt) { evt.Delete("json"); - evt.Delete("googlecloud.audit.request_metadata.requestAttributes"); - evt.Delete("googlecloud.audit.request_metadata.destinationAttributes"); }; - // Rename nested fields - var RenameNestedFields = function(evt) { + // Rename nested fields. + var renameNestedFields = function(evt) { var arr = evt.Get("googlecloud.audit.authorization_info"); for (var i = 0; i < arr.length; i++) { - arr[i].resource_attributes = arr[i].resourceAttributes; - delete arr[i].resourceAttributes; + if (arr[i].resourceAttributes) { + // Convert to snake_case. + arr[i].resource_attributes = arr[i].resourceAttributes; + delete arr[i].resourceAttributes; + } } }; // Set ECS categorization fields. var setECSCategorization = function(evt) { + evt.Put("event.kind", "event"); + + // google.rpc.Code value for OK is 0. + if (evt.Get("googlecloud.audit.status.code") === 0) { + evt.Put("event.outcome", "success"); + return; + } + + // Try to use authorization_info.granted when there was no status code. if (evt.Get("googlecloud.audit.status.code") == null) { var authorization_info = evt.Get("googlecloud.audit.authorization_info"); if (authorization_info.length === 1) { @@ -147,13 +275,11 @@ function Audit(keep_original_message) { } } else { evt.Put("event.outcome", "unknown"); - } - } else if (evt.Get("googlecloud.audit.status.code") === 0) { - evt.Put("event.outcome", "success"); - } else { - evt.Put("event.outcome", "failure"); + } + return; } - evt.Put("event.kind", "event"); + + evt.Put("event.outcome", "failure"); }; var pipeline = new processor.Chain() @@ -167,7 +293,7 @@ function Audit(keep_original_message) { .Add(convertProtoPayload) .Add(copyFields) .Add(dropExtraFields) - .Add(RenameNestedFields) + .Add(renameNestedFields) .Add(setECSCategorization) .Build(); diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log index 6446bf639f4..0eeac80b8e6 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log @@ -2,3 +2,5 @@ {"insertId":"-h6onuze1h7dg","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":false,"permission":"compute.machineTypes.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.machineTypes.aggregatedList","numResponseItems":"71","request":{"@type":"type.googleapis.com/compute.machineTypes.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:45:51.711Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/machineTypes","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:45:52.367887078Z","resource":{"labels":{"location":"global","method":"compute.machineTypes.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:45:51.228Z"} {"insertId":"yonau2dg2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"granted":true,"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"response":{"@type":"core.k8s.io/v1.Status","apiVersion":"v1","details":{"group":"batch","kind":"jobs","name":"gsuite-exporter-1589294700","uid":"2beff34a-945f-11ea-bacf-42010a80007f"},"kind":"Status","metadata":{},"status":"Success"},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"} {"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"} +{"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"} +{"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"} diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json index 8e5b00aeef8..55254263dc1 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json @@ -195,5 +195,107 @@ "user_agent.os.name": "Mac OS X", "user_agent.os.version": "10.15", "user_agent.version": "71.0." + }, + { + "@timestamp": "2020-08-05T21:07:30.974Z", + "cloud.project.id": "elastic-siem", + "event.action": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", + "event.dataset": "googlecloud.audit", + "event.id": "87efd529-6349-45d2-b905-fc607e6c5d3b", + "event.kind": "event", + "event.module": "googlecloud", + "event.outcome": "success", + "fileset.name": "audit", + "googlecloud.audit.authentication_info.principal_email": "system:serviceaccount:cert-manager:cert-manager-webhook", + "googlecloud.audit.authorization_info": [ + { + "granted": true, + "permission": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", + "resource": "authorization.k8s.io/v1beta1/subjectaccessreviews" + } + ], + "googlecloud.audit.method_name": "io.k8s.authorization.v1beta1.subjectaccessreviews.create", + "googlecloud.audit.request.proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "googlecloud.audit.request_metadata.caller_ip": "10.11.12.13", + "googlecloud.audit.request_metadata.caller_supplied_user_agent": "webhook/v0.0.0 (linux/amd64) kubernetes/$Format", + "googlecloud.audit.resource_name": "authorization.k8s.io/v1beta1/subjectaccessreviews", + "googlecloud.audit.response.proto_name": "authorization.k8s.io/v1beta1.SubjectAccessReview", + "googlecloud.audit.response.status": "map[allowed:true reason:RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\"]", + "googlecloud.audit.service_name": "k8s.io", + "googlecloud.audit.status.code": 0, + "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access", + "log.offset": 5100, + "service.name": "k8s.io", + "service.type": "googlecloud", + "source.ip": "10.11.12.13", + "tags": [ + "forwarded" + ], + "user.email": "system:serviceaccount:cert-manager:cert-manager-webhook", + "user_agent.device.name": "Other", + "user_agent.name": "Other", + "user_agent.original": "webhook/v0.0.0 (linux/amd64) kubernetes/$Format", + "user_agent.os.name": "Linux" + }, + { + "@timestamp": "2020-08-05T21:59:26.456Z", + "cloud.project.id": "foo", + "event.action": "v1.compute.images.insert", + "event.dataset": "googlecloud.audit", + "event.id": "v2spcwdzmc2", + "event.kind": "event", + "event.module": "googlecloud", + "event.outcome": "success", + "fileset.name": "audit", + "googlecloud.audit.authentication_info.principal_email": "user@mycompany.com", + "googlecloud.audit.authorization_info": [ + { + "granted": true, + "permission": "compute.images.create", + "resource_attributes": { + "name": "projects/foo/global/images/windows-server-2016-v20200805", + "service": "compute", + "type": "compute.images" + } + } + ], + "googlecloud.audit.method_name": "v1.compute.images.insert", + "googlecloud.audit.request.name": "windows-server-2016-v20200805", + "googlecloud.audit.request.proto_name": "type.googleapis.com/compute.images.insert", + "googlecloud.audit.request_metadata.caller_ip": "1.2.3.4", + "googlecloud.audit.request_metadata.caller_supplied_user_agent": "google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)", + "googlecloud.audit.resource_location.current_locations": [ + "eu" + ], + "googlecloud.audit.resource_name": "projects/foo/global/images/windows-server-2016-v20200805", + "googlecloud.audit.response.proto_name": "type.googleapis.com/operation", + "googlecloud.audit.response.status": "RUNNING", + "googlecloud.audit.service_name": "compute.googleapis.com", + "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity", + "log.offset": 7530, + "service.name": "compute.googleapis.com", + "service.type": "googlecloud", + "source.geo.city_name": "Moscow", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "RU", + "source.geo.location.lat": 55.7527, + "source.geo.location.lon": 37.6172, + "source.geo.region_iso_code": "RU-MOW", + "source.geo.region_name": "Moscow", + "source.ip": "1.2.3.4", + "tags": [ + "forwarded" + ], + "user.email": "user@mycompany.com", + "user_agent.device.name": "Mac", + "user_agent.name": "Other", + "user_agent.original": "google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)", + "user_agent.os.full": "Mac OS X 19.6.0", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "19.6.0" } ] \ No newline at end of file From f9e1ed94585cfad412a0982be72475594579b999 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 6 Aug 2020 08:57:54 -0400 Subject: [PATCH 2/2] Add test event without authorization_info[] --- CHANGELOG.next.asciidoc | 1 + .../googlecloud/audit/config/pipeline.js | 25 +++++----- .../audit/test/audit-log-entries.json.log | 1 + .../audit-log-entries.json.log-expected.json | 46 +++++++++++++++++++ 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d9e51467d70..39324ab452c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -233,6 +233,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245] - Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273] - Fix `okta` geoip lookup in pipeline for `destination.ip` {pull}20454[20454] +- Fix mapping exception in the `googlecloud/audit` dataset pipeline. {issue}18465[18465] {pull}20465[20465] *Heartbeat* diff --git a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js index dbfa0479a93..a24bd621934 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js +++ b/x-pack/filebeat/module/googlecloud/audit/config/pipeline.js @@ -243,11 +243,13 @@ function Audit(keep_original_message) { // Rename nested fields. var renameNestedFields = function(evt) { var arr = evt.Get("googlecloud.audit.authorization_info"); - for (var i = 0; i < arr.length; i++) { - if (arr[i].resourceAttributes) { - // Convert to snake_case. - arr[i].resource_attributes = arr[i].resourceAttributes; - delete arr[i].resourceAttributes; + if (Array.isArray(arr)) { + for (var i = 0; i < arr.length; i++) { + if (arr[i].resourceAttributes) { + // Convert to snake_case. + arr[i].resource_attributes = arr[i].resourceAttributes; + delete arr[i].resourceAttributes; + } } } }; @@ -265,17 +267,16 @@ function Audit(keep_original_message) { // Try to use authorization_info.granted when there was no status code. if (evt.Get("googlecloud.audit.status.code") == null) { var authorization_info = evt.Get("googlecloud.audit.authorization_info"); - if (authorization_info.length === 1) { - if (authorization_info[0].granted == null) { - evt.Put("event.outcome", "unknown"); - } else if (authorization_info[0].granted === true) { + if (Array.isArray(authorization_info) && authorization_info.length === 1) { + if (authorization_info[0].granted === true) { evt.Put("event.outcome", "success"); - } else { + } else if (authorization_info[0].granted === false) { evt.Put("event.outcome", "failure"); } - } else { - evt.Put("event.outcome", "unknown"); + return } + + evt.Put("event.outcome", "unknown"); return; } diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log index 0eeac80b8e6..9c228890527 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log @@ -4,3 +4,4 @@ {"insertId":"yonau3dc2zi","logName":"projects/elastic-beats/logs/cloudaudit.googleapis.com%2Fdata_access","protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"xxx@xxx.xxx"},"authorizationInfo":[{"permission":"compute.instances.list","resourceAttributes":{"name":"projects/elastic-beats","service":"resourcemanager","type":"resourcemanager.projects"}}],"methodName":"beta.compute.instances.aggregatedList","numResponseItems":"61","request":{"@type":"type.googleapis.com/compute.instances.aggregatedList"},"requestMetadata":{"callerIp":"192.168.1.1","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:71.0) Gecko/20100101 Firefox/71.0,gzip(gfe),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2019-12-19T00:44:25.198Z"}},"resourceLocation":{"currentLocations":["global"]},"resourceName":"projects/elastic-beats/global/instances","serviceName":"compute.googleapis.com","status":{"code":7,"message":"PERMISSION_DENIED"}},"receiveTimestamp":"2019-12-19T00:44:25.262379373Z","resource":{"labels":{"location":"global","method":"compute.instances.aggregatedList","project_id":"elastic-beats","service":"compute.googleapis.com","version":"beta"},"type":"api"},"severity":"INFO","timestamp":"2019-12-19T00:44:25.051Z"} {"insertId":"87efd529-6349-45d2-b905-fc607e6c5d3b","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"cert-manager-webhook:auth-delegator\" of ClusterRole \"system:auth-delegator\" to ServiceAccount \"cert-manager-webhook/cert-manager\""},"logName":"projects/foo/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"first":true,"id":"5555555-6349-45d2-b905-fc607e6c5d3b","last":true,"producer":"k8s.io"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:serviceaccount:cert-manager:cert-manager-webhook"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","resource":"authorization.k8s.io/v1beta1/subjectaccessreviews"}],"methodName":"io.k8s.authorization.v1beta1.subjectaccessreviews.create","request":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":false}},"requestMetadata":{"callerIp":"10.11.12.13","callerSuppliedUserAgent":"webhook/v0.0.0 (linux/amd64) kubernetes/$Format"},"resourceName":"authorization.k8s.io/v1beta1/subjectaccessreviews","response":{"@type":"authorization.k8s.io/v1beta1.SubjectAccessReview","apiVersion":"authorization.k8s.io/v1beta1","kind":"SubjectAccessReview","metadata":{"creationTimestamp":null},"spec":{"group":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"nonResourceAttributes":{"path":"/apis/webhook.cert-manager.io/v1beta1","verb":"get"},"user":"system:serviceaccount:kube-system:resourcequota-controller"},"status":{"allowed":true,"reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}},"serviceName":"k8s.io","status":{"code":0}},"receiveTimestamp":"2020-08-05T21:07:32.157698684Z","resource":{"labels":{"cluster_name":"analysis-cluster","location":"us-central1-a","project_id":"elastic-siem"},"type":"k8s_cluster"},"timestamp":"2020-08-05T21:07:30.974750Z"} {"insertId":"v2spcwdzmc2","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"first":true,"id":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"authorizationInfo":[{"granted":true,"permission":"compute.images.create","resourceAttributes":{"name":"projects/foo/global/images/windows-server-2016-v20200805","service":"compute","type":"compute.images"}}],"methodName":"v1.compute.images.insert","request":{"@type":"type.googleapis.com/compute.images.insert","family":"windows-server-2016","guestOsFeatures":[{"type":"VIRTIO_SCSI_MULTIQUEUE"},{"type":"WINDOWS"}],"name":"windows-server-2016-v20200805","rawDisk":{"source":"https://storage.googleapis.com/storage/v1/b/foo/o/windows-server-2016-v20200805.tar.gz"},"sourceType":"RAW"},"requestMetadata":{"callerIp":"1.2.3.4","callerSuppliedUserAgent":"google-cloud-sdk gcloud/290.0.1 command/gcloud.compute.images.create invocation-id/032752ad0fa44b4ea951951d2deef6a3 environment/None environment-version/None interactive/True from-script/False python/2.7.17 term/xterm-256color (Macintosh; Intel Mac OS X 19.6.0),gzip(gfe)","destinationAttributes":{},"requestAttributes":{"auth":{},"time":"2020-08-05T21:59:27.515Z"}},"resourceLocation":{"currentLocations":["eu"]},"resourceName":"projects/foo/global/images/windows-server-2016-v20200805","response":{"@type":"type.googleapis.com/operation","id":"44919313","insertTime":"2020-08-05T14:59:27.259-07:00","name":"operation-1596664766354-5ac287c395484-fa3923bd-543e018e","operationType":"insert","progress":"0","selfLink":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/operation-1596664766354-5ac287c395484-fa3923bd-543e018e","selfLinkWithId":"https://www.googleapis.com/compute/v1/projects/foo/global/operations/4491931805423146320","startTime":"2020-08-05T14:59:27.274-07:00","status":"RUNNING","targetId":"12345","targetLink":"https://www.googleapis.com/compute/v1/projects/foo/global/images/windows-server-2016-v20200805","user":"user@mycompany.com"},"serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T21:59:27.822546978Z","resource":{"labels":{"image_id":"771879043","project_id":"foo"},"type":"gce_image"},"severity":"NOTICE","timestamp":"2020-08-05T21:59:26.456Z"} +{"insertId":"-c7ctxmd2zab","logName":"projects/foo/logs/cloudaudit.googleapis.com%2Factivity","operation":{"id":"operation-1596646123456-5ac2438b775f6-f8ca1382-e70b6831","last":true,"producer":"compute.googleapis.com"},"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"user@mycompany.com"},"methodName":"beta.compute.instances.stop","request":{"@type":"type.googleapis.com/compute.instances.stop"},"requestMetadata":{"callerIp":"2.3.4.5","callerSuppliedUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)"},"resourceName":"projects/foo/zones/us-central1-a/instances/win10-test","serviceName":"compute.googleapis.com"},"receiveTimestamp":"2020-08-05T16:56:41.315135528Z","resource":{"labels":{"instance_id":"590261181","project_id":"foo","zone":"us-central1-a"},"type":"gce_instance"},"severity":"NOTICE","timestamp":"2020-08-05T16:56:40.428Z"} diff --git a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json index 55254263dc1..2d1832bc54a 100644 --- a/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json +++ b/x-pack/filebeat/module/googlecloud/audit/test/audit-log-entries.json.log-expected.json @@ -297,5 +297,51 @@ "user_agent.os.full": "Mac OS X 19.6.0", "user_agent.os.name": "Mac OS X", "user_agent.os.version": "19.6.0" + }, + { + "@timestamp": "2020-08-05T16:56:40.428Z", + "cloud.instance.id": "590261181", + "cloud.project.id": "foo", + "event.action": "beta.compute.instances.stop", + "event.dataset": "googlecloud.audit", + "event.id": "-c7ctxmd2zab", + "event.kind": "event", + "event.module": "googlecloud", + "event.outcome": "unknown", + "fileset.name": "audit", + "googlecloud.audit.authentication_info.principal_email": "user@mycompany.com", + "googlecloud.audit.method_name": "beta.compute.instances.stop", + "googlecloud.audit.request.proto_name": "type.googleapis.com/compute.instances.stop", + "googlecloud.audit.request_metadata.caller_ip": "2.3.4.5", + "googlecloud.audit.request_metadata.caller_supplied_user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)", + "googlecloud.audit.resource_name": "projects/foo/zones/us-central1-a/instances/win10-test", + "googlecloud.audit.service_name": "compute.googleapis.com", + "googlecloud.audit.type": "type.googleapis.com/google.cloud.audit.AuditLog", + "input.type": "log", + "log.logger": "projects/foo/logs/cloudaudit.googleapis.com%2Factivity", + "log.offset": 9946, + "service.name": "compute.googleapis.com", + "service.type": "googlecloud", + "source.as.number": 3215, + "source.as.organization.name": "Orange", + "source.geo.city_name": "Clermont-Ferrand", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "FR", + "source.geo.location.lat": 45.7838, + "source.geo.location.lon": 3.0966, + "source.geo.region_iso_code": "FR-63", + "source.geo.region_name": "Puy-de-D\u00f4me", + "source.ip": "2.3.4.5", + "tags": [ + "forwarded" + ], + "user.email": "user@mycompany.com", + "user_agent.device.name": "Mac", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:79.0) Gecko/20100101 Firefox/79.0,gzip(gfe),gzip(gfe)", + "user_agent.os.full": "Mac OS X 10.15", + "user_agent.os.name": "Mac OS X", + "user_agent.os.version": "10.15", + "user_agent.version": "79.0." } ] \ No newline at end of file