From eeb5d2356d48eaea4ad6b1e10beda9cf82bbca27 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Fri, 12 Feb 2021 10:35:30 +0100 Subject: [PATCH] [ecs] Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline (#23929) * Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline * Add description field and set _id properly --- CHANGELOG.next.asciidoc | 1 + .../module/okta/system/config/input.yml | 16 +- .../module/okta/system/config/pipeline.js | 215 -------- .../module/okta/system/ingest/pipeline.yml | 497 ++++++++++++++++++ .../okta-system-test.json.log-expected.json | 41 +- 5 files changed, 538 insertions(+), 232 deletions(-) delete mode 100644 x-pack/filebeat/module/okta/system/config/pipeline.js diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 36e55e70efb..0896a810b25 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -846,6 +846,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Upgrade sophos/xg fileset to ECS 1.8.0. {issue}23118[23118] {pull}23967[23967] - Upgrade system/auth to ECS 1.8 {issue}23118[23118] {pull}23961[23961] - Upgrade elasticsearch/audit to ECS 1.8 {issue}23118[23118] {pull}24000[24000] +- Upgrade okta to ecs 1.8.0 and move js processor to ingest pipeline {issue}23118[23118] {pull}23929[23929] *Heartbeat* diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index f2a8e5fe906..8612068d54d 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -50,12 +50,16 @@ tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: - - script: - lang: javascript - id: okta_system_script - file: ${path.home}/module/okta/system/config/pipeline.js - params: - keep_original_message: {{ .keep_original_message }} + - decode_json_fields: + fields: + - message + target: json +{{ if eq .keep_original_message true }} + - rename: + fields: + - from: message + to: event.original +{{ end }} - add_fields: target: '' fields: diff --git a/x-pack/filebeat/module/okta/system/config/pipeline.js b/x-pack/filebeat/module/okta/system/config/pipeline.js deleted file mode 100644 index 0d381b0944d..00000000000 --- a/x-pack/filebeat/module/okta/system/config/pipeline.js +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -function OktaSystem(keep_original_message) { - var processor = require("processor"); - - var decodeJson = new processor.DecodeJSONFields({ - fields: ["message"], - target: "json", - }); - - var setId = function(evt) { - var oktaUuid = evt.Get("json.uuid"); - if (oktaUuid) { - evt.Put("@metadata._id", oktaUuid); - } - }; - - var parseTimestamp = new processor.Timestamp({ - field: "json.published", - timezone: "UTC", - layouts: ["2006-01-02T15:04:05.999Z"], - tests: ["2020-02-05T18:19:23.599Z"], - ignore_missing: true, - }); - - var saveOriginalMessage = function(evt) {}; - if (keep_original_message) { - saveOriginalMessage = new processor.Convert({ - fields: [ - {from: "message", to: "event.original"} - ], - mode: "rename" - }); - } - - var dropOriginalMessage = function(evt) { - evt.Delete("message"); - }; - - var categorizeEvent = new processor.AddFields({ - target: "event", - fields: { - category: ["authentication"], - kind: "event", - type: ["access"], - - }, - }); - - var convertFields = new processor.Convert({ - fields: [ - { from: "json.displayMessage", to: "okta.display_message" }, - { from: "json.eventType", to: "okta.event_type" }, - { from: "json.uuid", to: "okta.uuid" }, - { from: "json.actor.alternateId", to: "okta.actor.alternate_id" }, - { from: "json.actor.displayName", to: "okta.actor.display_name" }, - { from: "json.actor.id", to: "okta.actor.id" }, - { from: "json.actor.type", to: "okta.actor.type" }, - { from: "json.client.device", to: "okta.client.device" }, - { from: "json.client.geographicalContext.geolocation", to: "client.geo.location" }, - { from: "json.client.geographicalContext.city", to: "client.geo.city_name" }, - { from: "json.client.geographicalContext.state", to: "client.geo.region_name" }, - { from: "json.client.geographicalContext.country", to: "client.geo.country_name" }, - { from: "json.client.id", to: "okta.client.id" }, - { from: "json.client.ipAddress", to: "okta.client.ip" }, - { from: "json.client.userAgent.browser", to: "okta.client.user_agent.browser" }, - { from: "json.client.userAgent.os", to: "okta.client.user_agent.os" }, - { from: "json.client.userAgent.rawUserAgent", to: "okta.client.user_agent.raw_user_agent" }, - { from: "json.client.zone", to: "okta.client.zone" }, - { from: "json.outcome.reason", to: "okta.outcome.reason" }, - { from: "json.outcome.result", to: "okta.outcome.result" }, - { from: "json.target", to: "okta.target" }, - { from: "json.transaction.id", to: "okta.transaction.id" }, - { from: "json.transaction.type", to: "okta.transaction.type" }, - { from: "json.debugContext.debugData.deviceFingerprint", to: "okta.debug_context.debug_data.device_fingerprint" }, - { from: "json.debugContext.debugData.requestId", to: "okta.debug_context.debug_data.request_id" }, - { from: "json.debugContext.debugData.requestUri", to: "okta.debug_context.debug_data.request_uri" }, - { from: "json.debugContext.debugData.threatSuspected", to: "okta.debug_context.debug_data.threat_suspected" }, - { from: "json.debugContext.debugData.url", to: "okta.debug_context.debug_data.url" }, - { from: "json.authenticationContext.authenticationProvider", to: "okta.authentication_context.authentication_provider" }, - { from: "json.authenticationContext.authenticationStep", to: "okta.authentication_context.authentication_step" }, - { from: "json.authenticationContext.credentialProvider", to: "okta.authentication_context.credential_provider" }, - { from: "json.authenticationContext.credentialType", to: "okta.authentication_context.credential_type" }, - { from: "json.authenticationContext.externalSessionId", to: "okta.authentication_context.external_session_id" }, - { from: "json.authenticationContext.interface", to: "okta.authentication_context.authentication_provider" }, - { from: "json.authenticationContext.issuer", to: "okta.authentication_context.issuer" }, - { from: "json.securityContext.asNumber", to: "okta.security_context.as.number" }, - { from: "json.securityContext.asOrg", to: "okta.security_context.as.organization.name" }, - { from: "json.securityContext.domain", to: "okta.security_context.domain" }, - { from: "json.securityContext.isProxy", to: "okta.security_context.is_proxy" }, - { from: "json.securityContext.isp", to: "okta.security_context.isp" }, - ], - mode: "rename", - ignore_missing: true, - fail_on_error: false, - }); - - var copyFields = new processor.Convert({ - fields: [ - { from: "okta.client.user_agent.raw_user_agent", to: "user_agent.original" }, - { from: "okta.client.ip", to: "client.ip" }, - { from: "okta.client.ip", to: "source.ip" }, - { from: "okta.event_type", to: "event.action" }, - { from: "okta.security_context.as.number", to: "client.as.number" }, - { from: "okta.security_context.as.organization.name", to: "client.as.organization.name" }, - { from: "okta.security_context.domain", to: "client.domain" }, - { from: "okta.security_context.domain", to: "source.domain" }, - { from: "okta.uuid", to: "event.id" }, - ], - ignore_missing: true, - fail_on_error: false, - }); - - var setEventOutcome = function(evt) { - var outcome = evt.Get("okta.outcome.result"); - if (outcome) { - outcome = outcome.toLowerCase(); - if (outcome === "success" || outcome === "allow") { - evt.Put("event.outcome", "success"); - } else if (outcome === "failure" || outcome === "deny") { - evt.Put("event.outcome", "failure"); - } else { - evt.Put("event.outcome", "unknown"); - } - } - }; - - // Update nested fields - var renameNestedFields = function(evt) { - var arr = evt.Get("okta.target"); - if (arr) { - for (var i = 0; i < arr.length; i++) { - arr[i].alternate_id = arr[i].alternateId; - arr[i].display_name = arr[i].displayName; - delete arr[i].alternateId; - delete arr[i].displayName; - delete arr[i].detailEntry; - } - } - }; - - // Set user info if actor type is User - var setUserInfo = function(evt) { - if (evt.Get("okta.actor.type") === "User") { - evt.Put("client.user.full_name", evt.Get("okta.actor.display_name")); - evt.Put("source.user.full_name", evt.Get("okta.actor.display_name")); - evt.Put("related.user", evt.Get("okta.actor.display_name")); - evt.Put("client.user.id", evt.Get("okta.actor.id")); - evt.Put("source.user.id", evt.Get("okta.actor.id")); - } - }; - - // Set related.ip field - var setRelatedIP = function(event) { - var ip = event.Get("source.ip"); - if (ip) { - event.AppendTo("related.ip", ip); - } - ip = event.Get("destination.ip"); - if (ip) { - event.AppendTo("related.ip", ip); - } - }; - - // Drop extra fields - var dropExtraFields = function(evt) { - evt.Delete("json"); - }; - - // Remove null fields - var dropNullFields = function(evt) { - function dropNull(obj) { - Object.keys(obj).forEach(function(key) { - (obj[key] && typeof obj[key] === 'object') && dropNull(obj[key]) || - (obj[key] === null) && delete obj[key]; - }); - return obj; - } - dropNull(evt); - }; - - var pipeline = new processor.Chain() - .Add(decodeJson) - .Add(setId) - .Add(parseTimestamp) - .Add(saveOriginalMessage) - .Add(dropOriginalMessage) - .Add(categorizeEvent) - .Add(convertFields) - .Add(copyFields) - .Add(setEventOutcome) - .Add(renameNestedFields) - .Add(setUserInfo) - .Add(setRelatedIP) - .Add(dropExtraFields) - .Add(dropNullFields) - .Build(); - - return { - process: pipeline.Run, - }; -} - -var oktaSystem; - -// Register params from configuration. -function register(params) { - oktaSystem = new OktaSystem(params.keep_original_message); -} - -function process(evt) { - return oktaSystem.process(evt); -} diff --git a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml index 0da85185ca2..dc576e9c70c 100644 --- a/x-pack/filebeat/module/okta/system/ingest/pipeline.yml +++ b/x-pack/filebeat/module/okta/system/ingest/pipeline.yml @@ -4,6 +4,503 @@ processors: - set: field: event.ingested value: "{{_ingest.timestamp}}" + - script: + description: Drops null/empty values recursively + lang: painless + source: | + boolean drop(Object o) { + if (o == null || o == "") { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); + - remove: + field: message + ignore_missing: true + - convert: + field: json.uuid + target_field: _id + type: string + ignore_failure: true + if: ctx?.json?.uuid != null && ctx?.json?.uuid != "" + - date: + field: json.published + formats: + - ISO8601 + ignore_failure: true + - set: + field: event.kind + value: event + - rename: + field: json.displayMessage + target_field: okta.display_message + ignore_missing: true + ignore_failure: true + - rename: + field: json.eventType + target_field: okta.event_type + ignore_missing: true + ignore_failure: true + - append: + field: event.category + value: iam + if: | + ["group.user_membership.add","group.user_membership.remove", + "user.lifecycle.activate","user.lifecycle.create", + "user.lifecycle.deactivate","user.lifecycle.suspend", + "user.lifecycle.unsuspend"].contains(ctx?.okta?.event_type) + - append: + field: event.category + value: configuration + if: | + ["policy.lifecycle.activate","policy.lifecycle.create", + "policy.lifecycle.deactivate","policy.lifecycle.delete", + "policy.lifecycle.update","policy.rule.activate","policy.rule.add", + "policy.rule.deactivate","policy.rule.delete", + "application.lifecycle.create","application.lifecycle.delete", + "policy.rule.update","application.lifecycle.activate", + "application.lifecycle.deactivate","application.lifecycle.update"].contains(ctx?.okta?.event_type) + - append: + field: event.category + value: authentication + if: '["user.session.start","user.session.end","user.authentication.sso","policy.evaluate_sign_on"].contains(ctx?.okta?.event_type)' + - append: + field: event.category + value: session + if: '["user.session.start","user.session.end"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: info + if: | + ["system.org.rate_limit.warning","system.org.rate_limit.violation", + "core.concurrency.org.limit.violation"].contains(ctx?.okta?.event_type) + - append: + field: event.type + value: network + if: '["security.request.blocked"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: network + if: | + ["system.org.rate_limit.warning","system.org.rate_limit.violation", + "core.concurrency.org.limit.violation","security.request.blocked"].contains(ctx?.okta?.event_type) + - append: + field: event.type + value: start + if: '["user.session.start"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: end + if: '["user.session.end"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: group + if: '["group.user_membership.add","group.user_membership.remove"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: user + if: | + ["user.lifecycle.activate","user.lifecycle.create", + "user.lifecycle.deactivate","user.lifecycle.suspend", + "user.lifecycle.unsuspend","user.authentication.sso", + "user.session.start","user.session.end","application.user_membership.add", + "application.user_membership.remove","application.user_membership.change_username"].contains(ctx?.okta?.event_type) + - append: + field: event.type + value: change + if: | + ["user.lifecycle.activate","user.lifecycle.deactivate", + "user.lifecycle.suspend","user.lifecycle.unsuspend", + "group.user_membership.add","group.user_membership.remove", + "policy.lifecycle.activate","policy.lifecycle.deactivate", + "policy.lifecycle.update","policy.rule.activate","policy.rule.add", + "policy.rule.deactivate","policy.rule.update","application.user_membership.add", + "application.user_membership.remove","application.user_membership.change_username"].contains(ctx?.okta?.event_type) + - append: + field: event.type + value: creation + if: '["user.lifecycle.create","policy.lifecycle.create","application.lifecycle.create"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: deletion + if: '["policy.lifecycle.delete","application.lifecycle.delete"].contains(ctx?.okta?.event_type)' + - append: + field: event.type + value: info + if: '["policy.evaluate_sign_on"].contains(ctx?.okta?.event_type)' + - rename: + field: json.uuid + target_field: okta.uuid + ignore_missing: true + ignore_failure: true + - rename: + field: json.actor.alternateId + target_field: okta.actor.alternate_id + ignore_missing: true + ignore_failure: true + - rename: + field: json.actor.displayName + target_field: okta.actor.display_name + ignore_missing: true + ignore_failure: true + - rename: + field: json.actor.id + target_field: okta.actor.id + ignore_missing: true + ignore_failure: true + - rename: + field: json.actor.type + target_field: okta.actor.type + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.device + target_field: okta.client.device + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.geographicalContext.geolocation + target_field: client.geo.location + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.geographicalContext.city + target_field: client.geo.city_name + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.geographicalContext.state + target_field: client.geo.region_name + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.geographicalContext.country + target_field: client.geo.country_name + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.id + target_field: okta.client.id + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.ipAddress + target_field: okta.client.ip + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.userAgent.browser + target_field: okta.client.user_agent.browser + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.userAgent.os + target_field: okta.client.user_agent.os + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.userAgent.rawUserAgent + target_field: okta.client.user_agent.raw_user_agent + ignore_missing: true + ignore_failure: true + - rename: + field: json.client.zone + target_field: okta.client.zone + ignore_missing: true + ignore_failure: true + - rename: + field: json.outcome.reason + target_field: okta.outcome.reason + ignore_missing: true + ignore_failure: true + - rename: + field: json.outcome.result + target_field: okta.outcome.result + ignore_missing: true + ignore_failure: true + - rename: + field: json.target + target_field: okta.target + ignore_missing: true + ignore_failure: true + - rename: + field: json.transaction.id + target_field: okta.transaction.id + ignore_missing: true + ignore_failure: true + - rename: + field: json.transaction.type + target_field: okta.transaction.type + ignore_missing: true + ignore_failure: true + - rename: + field: json.debugContext.debugData.deviceFingerprint + target_field: okta.debug_context.debug_data.device_fingerprint + ignore_missing: true + ignore_failure: true + - rename: + field: json.debugContext.debugData.requestId + target_field: okta.debug_context.debug_data.request_id + ignore_missing: true + ignore_failure: true + - rename: + field: json.debugContext.debugData.requestUri + target_field: okta.debug_context.debug_data.request_uri + ignore_missing: true + ignore_failure: true + - rename: + field: json.debugContext.debugData.threatSuspected + target_field: okta.debug_context.debug_data.threat_suspected + ignore_missing: true + ignore_failure: true + - rename: + field: json.debugContext.debugData.url + target_field: okta.debug_context.debug_data.url + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.authenticationProvider + target_field: okta.authentication_context.authentication_provider + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.authenticationStep + target_field: okta.authentication_context.authentication_step + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.credentialProvider + target_field: okta.authentication_context.credential_provider + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.credentialType + target_field: okta.authentication_context.credential_type + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.externalSessionId + target_field: okta.authentication_context.external_session_id + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.interface + target_field: okta.authentication_context.authentication_provider + ignore_missing: true + ignore_failure: true + - rename: + field: json.authenticationContext.issuer + target_field: okta.authentication_context.issuer + ignore_missing: true + ignore_failure: true + - rename: + field: json.securityContext.asNumber + target_field: okta.security_context.as.number + ignore_missing: true + ignore_failure: true + - rename: + field: json.securityContext.asOrg + target_field: okta.security_context.as.organization.name + ignore_missing: true + ignore_failure: true + - rename: + field: json.securityContext.domain + target_field: okta.security_context.domain + ignore_missing: true + ignore_failure: true + - rename: + field: json.securityContext.isProxy + target_field: okta.security_context.is_proxy + ignore_missing: true + ignore_failure: true + - rename: + field: json.securityContext.isp + target_field: okta.security_context.isp + ignore_missing: true + ignore_failure: true + - convert: + field: okta.client.user_agent.raw_user_agent + target_field: user_agent.original + type: string + ignore_failure: true + - convert: + field: okta.client.ip + target_field: client.ip + type: string + ignore_failure: true + - convert: + field: okta.client.ip + target_field: source.ip + type: string + ignore_failure: true + - convert: + field: okta.event_type + target_field: event.action + type: string + ignore_failure: true + - convert: + field: okta.security_context.as.number + target_field: client.as.number + type: string + ignore_failure: true + - convert: + field: okta.security_context.as.organization.name + target_field: client.as.organization.name + type: string + ignore_failure: true + - convert: + field: okta.security_context.domain + target_field: client.domain + type: string + ignore_failure: true + - convert: + field: okta.security_context.domain + target_field: source.domain + type: string + ignore_failure: true + - convert: + field: okta.uuid + target_field: event.id + type: string + ignore_failure: true + - lowercase: + field: okta.outcome.result + target_field: okta.outcome.result_lower + ignore_missing: true + - set: + field: event.outcome + value: success + if: ctx?.okta?.outcome?.result_lower != null && (ctx?.okta?.outcome?.result_lower == "success" || ctx?.okta?.outcome?.result_lower == "allow") + - set: + field: event.outcome + value: failure + if: ctx?.okta?.outcome?.result_lower != null && (ctx?.okta?.outcome?.result_lower == "failure" || ctx?.okta?.outcome?.result_lower == "deny") + - set: + field: event.outcome + value: unknown + if: ctx?.event?.outcome == null + - remove: + field: okta.outcome.result_lower + ignore_missing: true + - script: + lang: painless + source: | + def arr = ctx?.okta?.target; + if (arr != null) { + for (def i = 0; i < arr.length; i++) { + arr[i]["alternate_id"] = arr[i]["alternateId"]; + arr[i].remove("alternateId"); + arr[i]["display_name"] = arr[i]["displayName"]; + arr[i].remove("displayName"); + arr[i].remove("detailEntry"); + } + } + - script: + lang: painless + source: | + def arr = ctx?.okta?.target; + if (arr != null) { + for (def i = 0; i < arr.length; i++) { + if (arr[i]["type"].toLowerCase().contains("user")) { + ctx["okta_target_user"] = arr[i]; + break; + } + } + } + if: ctx?.okta?.event_type != null && ctx?.okta?.event_type.contains("user.") + - script: + lang: painless + source: | + def arr = ctx?.okta?.target; + if (arr != null) { + for (def i = 0; i < arr.length; i++) { + if (arr[i]["type"].toLowerCase().contains("group")) { + ctx["okta_target_group"] = arr[i]; + break; + } + } + } + if: ctx?.okta?.event_type != null && ctx?.okta?.event_type.contains("group.") + - rename: + field: okta_target_user.display_name + target_field: user.target.full_name + ignore_missing: true + - rename: + field: okta_target_user.id + target_field: user.target.id + ignore_missing: true + - rename: + field: okta_target_user.login + target_field: user.target.email + ignore_missing: true + - rename: + field: okta_target_group.display_name + target_field: user.target.group.name + ignore_missing: true + - rename: + field: okta_target_group.id + target_field: user.target.group.id + ignore_missing: true + - remove: + field: + - okta_target_user + - okta_target_group + ignore_missing: true + - set: + field: client.user.id + value: "{{okta.actor.id}}" + ignore_empty_value: true + if: ctx?.okta?.actor?.id != null + - set: + field: source.user.id + value: "{{okta.actor.id}}" + ignore_empty_value: true + if: ctx?.okta?.actor?.id != null + - set: + field: client.user.full_name + value: "{{okta.actor.display_name}}" + ignore_empty_value: true + if: ctx?.okta?.actor?.display_name != null + - set: + field: source.user.full_name + value: "{{okta.actor.display_name}}" + ignore_empty_value: true + if: ctx?.okta?.actor?.display_name != null + - set: + field: user.full_name + value: "{{okta.actor.display_name}}" + ignore_empty_value: true + if: ctx?.okta?.actor?.display_name != null + - append: + field: related.user + value: "{{okta.actor.display_name}}" + allow_duplicates: false + if: ctx?.okta?.actor?.display_name != null + - append: + field: related.user + value: "{{user.target.full_name}}" + allow_duplicates: false + if: ctx?.user?.target?.full_name != null + - append: + field: related.ip + value: "{{source.ip}}" + allow_duplicates: false + if: ctx?.source?.ip != null + - append: + field: related.ip + value: "{{destination.ip}}" + allow_duplicates: false + if: ctx?.destination?.ip != null + - remove: + field: json + ignore_missing: true - user_agent: field: user_agent.original ignore_missing: true diff --git a/x-pack/filebeat/module/okta/system/test/okta-system-test.json.log-expected.json b/x-pack/filebeat/module/okta/system/test/okta-system-test.json.log-expected.json index 39d00244185..226b52efa7d 100644 --- a/x-pack/filebeat/module/okta/system/test/okta-system-test.json.log-expected.json +++ b/x-pack/filebeat/module/okta/system/test/okta-system-test.json.log-expected.json @@ -11,7 +11,8 @@ "client.user.id": "00u1abvz4pYqdM8ms4x6", "event.action": "user.session.end", "event.category": [ - "authentication" + "authentication", + "session" ], "event.dataset": "okta.system", "event.id": "faf7398a-4f77-11ea-97fb-5925e98228bd", @@ -20,7 +21,8 @@ "event.original": "{\"actor\":{\"alternateId\":\"xxxxxx@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102nZHzd6OHSfGG51vsoc22gw\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"108.255.197.247\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"authnRequestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestId\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"requestUri\":\"/login/signout\",\"threatSuspected\":\"false\",\"url\":\"/login/signout?message=login_page_messages.session_has_expired\"}},\"displayMessage\":\"User logout from Okta\",\"eventType\":\"user.session.end\",\"legacyEventType\":\"core.user_auth.logout_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T22:18:51.843Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"108.255.197.247\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkccyyMli2Uay2I93ZgRzQAAB0c\",\"type\":\"WEB\"},\"uuid\":\"faf7398a-4f77-11ea-97fb-5925e98228bd\",\"version\":\"0\"}", "event.outcome": "success", "event.type": [ - "access" + "end", + "user" ], "fileset.name": "system", "input.type": "log", @@ -47,8 +49,12 @@ "okta.transaction.id": "XkccyyMli2Uay2I93ZgRzQAAB0c", "okta.transaction.type": "WEB", "okta.uuid": "faf7398a-4f77-11ea-97fb-5925e98228bd", - "related.ip": "108.255.197.247", - "related.user": "xxxxxx", + "related.ip": [ + "108.255.197.247" + ], + "related.user": [ + "xxxxxx" + ], "service.type": "okta", "source.as.number": 7018, "source.as.organization.name": "AT&T Services, Inc.", @@ -66,6 +72,7 @@ "tags": [ "forwarded" ], + "user.full_name": "xxxxxx", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -86,7 +93,8 @@ "client.user.id": "00u1abvz4pYqdM8ms4x6", "event.action": "user.session.start", "event.category": [ - "authentication" + "authentication", + "session" ], "event.dataset": "okta.system", "event.id": "3aeede38-4f67-11ea-abd3-1f5d113f2546", @@ -95,7 +103,8 @@ "event.original": "{\"actor\":{\"alternateId\":\"xxxxxx@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102bZDNFfWaQSyEZQuDgWt-uQ\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"108.255.197.247\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"deviceFingerprint\":\"541daf91d15bef64a7e08c946fd9a9d0\",\"requestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestUri\":\"/api/v1/authn\",\"threatSuspected\":\"false\",\"url\":\"/api/v1/authn?\"}},\"displayMessage\":\"User login to Okta\",\"eventType\":\"user.session.start\",\"legacyEventType\":\"core.user_auth.login_success\",\"outcome\":{\"reason\":null,\"result\":\"SUCCESS\"},\"published\":\"2020-02-14T20:18:57.718Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"108.255.197.247\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":null,\"transaction\":{\"detail\":{},\"id\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"type\":\"WEB\"},\"uuid\":\"3aeede38-4f67-11ea-abd3-1f5d113f2546\",\"version\":\"0\"}", "event.outcome": "success", "event.type": [ - "access" + "start", + "user" ], "fileset.name": "system", "input.type": "log", @@ -123,8 +132,12 @@ "okta.transaction.id": "XkcAsWb8WjwDP76xh@1v8wAABp0", "okta.transaction.type": "WEB", "okta.uuid": "3aeede38-4f67-11ea-abd3-1f5d113f2546", - "related.ip": "108.255.197.247", - "related.user": "xxxxxx", + "related.ip": [ + "108.255.197.247" + ], + "related.user": [ + "xxxxxx" + ], "service.type": "okta", "source.as.number": 7018, "source.as.organization.name": "AT&T Services, Inc.", @@ -142,6 +155,7 @@ "tags": [ "forwarded" ], + "user.full_name": "xxxxxx", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0", @@ -171,7 +185,7 @@ "event.original": "{\"actor\":{\"alternateId\":\"xxxxxx@elastic.co\",\"detailEntry\":null,\"displayName\":\"xxxxxx\",\"id\":\"00u1abvz4pYqdM8ms4x6\",\"type\":\"User\"},\"authenticationContext\":{\"authenticationProvider\":null,\"authenticationStep\":0,\"credentialProvider\":null,\"credentialType\":null,\"externalSessionId\":\"102bZDNFfWaQSyEZQuDgWt-uQ\",\"interface\":null,\"issuer\":null},\"client\":{\"device\":\"Computer\",\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"id\":null,\"ipAddress\":\"108.255.197.247\",\"userAgent\":{\"browser\":\"FIREFOX\",\"os\":\"Mac OS X\",\"rawUserAgent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0\"},\"zone\":\"null\"},\"debugContext\":{\"debugData\":{\"deviceFingerprint\":\"541daf91d15bef64a7e08c946fd9a9d0\",\"requestId\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"requestUri\":\"/api/v1/authn\",\"threatSuspected\":\"false\",\"url\":\"/api/v1/authn?\"}},\"displayMessage\":\"Evaluation of sign-on policy\",\"eventType\":\"policy.evaluate_sign_on\",\"legacyEventType\":null,\"outcome\":{\"reason\":\"Sign-on policy evaluation resulted in ALLOW\",\"result\":\"ALLOW\"},\"published\":\"2020-02-14T20:18:57.762Z\",\"request\":{\"ipChain\":[{\"geographicalContext\":{\"city\":\"Dublin\",\"country\":\"United States\",\"geolocation\":{\"lat\":37.7201,\"lon\":-121.919},\"postalCode\":\"94568\",\"state\":\"California\"},\"ip\":\"108.255.197.247\",\"source\":null,\"version\":\"V4\"}]},\"securityContext\":{\"asNumber\":null,\"asOrg\":null,\"domain\":null,\"isProxy\":null,\"isp\":null},\"severity\":\"INFO\",\"target\":[{\"alternateId\":\"unknown\",\"detailEntry\":{\"policyType\":\"OktaSignOn\"},\"displayName\":\"Default Policy\",\"id\":\"00p1abvweGGDW10Ur4x6\",\"type\":\"PolicyEntity\"},{\"alternateId\":\"00p1abvweGGDW10Ur4x6\",\"detailEntry\":null,\"displayName\":\"Default Rule\",\"id\":\"0pr1abvwfqGFI4n064x6\",\"type\":\"PolicyRule\"}],\"transaction\":{\"detail\":{},\"id\":\"XkcAsWb8WjwDP76xh@1v8wAABp0\",\"type\":\"WEB\"},\"uuid\":\"3af594f9-4f67-11ea-abd3-1f5d113f2546\",\"version\":\"0\"}", "event.outcome": "success", "event.type": [ - "access" + "info" ], "fileset.name": "system", "input.type": "log", @@ -214,8 +228,12 @@ "okta.transaction.id": "XkcAsWb8WjwDP76xh@1v8wAABp0", "okta.transaction.type": "WEB", "okta.uuid": "3af594f9-4f67-11ea-abd3-1f5d113f2546", - "related.ip": "108.255.197.247", - "related.user": "xxxxxx", + "related.ip": [ + "108.255.197.247" + ], + "related.user": [ + "xxxxxx" + ], "service.type": "okta", "source.as.number": 7018, "source.as.organization.name": "AT&T Services, Inc.", @@ -233,6 +251,7 @@ "tags": [ "forwarded" ], + "user.full_name": "xxxxxx", "user_agent.device.name": "Mac", "user_agent.name": "Firefox", "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:72.0) Gecko/20100101 Firefox/72.0",