From 938e66ccf352e2925b99fc988b28c46fc3ee304e Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 23 Jul 2020 11:01:08 +0200 Subject: [PATCH 01/19] mofidy doc --- .../module/azure/app_insights/_meta/docs.asciidoc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc index 2ba1150078b..2b587acbbdd 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/docs.asciidoc @@ -45,17 +45,15 @@ This value is only valid when segment is specified. `filter`:: (_string_) An expression used to filter the results. This value should be a valid OData filter expression where the keys of each clause should be applicable dimensions for the metric you are retrieving. -Users can select the options to retrieve all metrics from a specific namespace using the following: +Example configuration: ["source","yaml"] ---- - metrics: - - id: ["*"] - timespan: "Microsoft.Storage/storageAccounts" +metrics: + - id: ["requests/count", "requests/failed"] + segment: "request/name" + aggregation: ["sum"] ---- -A default non configurable timegrain of 5 min is set so users are advised to configure an interval of 300s or a multiply of it. - - From 54f70e1b75f337cdca121c8234dfb8c6a15cc3d8 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Nov 2020 10:18:56 +0100 Subject: [PATCH 02/19] platformlogs --- x-pack/filebeat/filebeat.reference.yml | 10 ++ x-pack/filebeat/module/azure/_meta/config.yml | 10 ++ .../filebeat/module/azure/_meta/docs.asciidoc | 13 ++ x-pack/filebeat/module/azure/fields.go | 2 +- .../azure/platformlogs/_meta/fields.yml | 45 +++++ .../platformlogs/config/azure-eventhub.yml | 16 ++ .../module/azure/platformlogs/config/file.yml | 14 ++ .../azure/platformlogs/ingest/pipeline.yml | 165 ++++++++++++++++++ .../module/azure/platformlogs/manifest.yml | 20 +++ .../azure/platformlogs/test/platformlogs.log | 1 + .../test/platformlogs.log-expected.json | 67 +++++++ .../test/supporttickets_write.log | 1 + .../supporttickets_write.log-expected.json | 77 ++++++++ x-pack/filebeat/modules.d/azure.yml.disabled | 10 ++ 14 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml create mode 100644 x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml create mode 100644 x-pack/filebeat/module/azure/platformlogs/config/file.yml create mode 100644 x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/azure/platformlogs/manifest.yml create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index ab4eb1e6acd..a47280871a8 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -365,6 +365,16 @@ filebeat.modules: # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index ab7f477b8bb..588cd3c3078 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -14,6 +14,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index aa5c854b457..55d90215aa5 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -19,6 +19,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -40,6 +43,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index d358caa2edd..f3358cf79e8 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW01z2zYQvftX7PiYady7Dp1RnaTjmSbOOM6ZA5MrGjVEsAAoj/LrO+D3BwBCFkil0+rUmsq+R2B3sfsWeg8veNwA+VEIvAJQVDHcwPVW///1FYBAhkTiBp5QkSuABGUsaK4ozzbw2xUAQPld+MyTgmkTO4oskZvy0XvIyB478/qjjjluIBW8yOu/GGwOzfRNyeKp/XZEk/Z5Y/gFj69c9P9uNF99Kup9k3D3YQIZcyGQkSCIt50tE5TCjGTqbJTKjAlAoOSFiHFiv78hM9YfpjbGu9WHHLyM64VmYPvQ/VfrY43fIhTc1G6DmAt+oAmKBUC1jV81iszJYLU7dPPTVeHDIX8ZW2vTR6GeuaA/qhAUVZ4JBLrt24aB7RY+VvRA1ZHxVM7GzThhdjR2pGAqKiNlAzvCJPqF26cytmDHRZ2vGj6gCd14RyFmiqqjceVMYTOzbndmeyYKfRoxI3QvI5pRRYnCJHo6RoWchI+bmgc9/bktsaDFgqcjWLBstMHt7EOqZv87gTCYgmBMI6UHzNbh8ocTqjuRxTp0vjmAGjK7grF12HxyIbVrEz/jnqywNGacYdTdvLNGGX/6C2NleFw9iOao9r4W7Ume0yyt/831u+tzotf6SoMzYYnssZ0B8EkZMub5GoFhhekfYca3CMxka8dpqKAul7JJxTIkY9u3E6h8dOO4tq9PV3CGEZGSptkeMxW5thQ8F/OEt9CfB84QOgpOrxoRT3BXHn/jzmUt1h3+uGK3UV6f5MkucJGV7O2/x0rmgmYxzQlbnezXBvk0mprJpYhOsBt6PEdR9RxhG537xu60puz16bpbMCzLea1dwZT9fWtQ7WhEdYpNOGSz5bZKIQpTLswtyptgb00Wu4NIB3N40I/arhs6F9q1FMVxffPmduyrzeJcQyZRHGiMkcC/C5SW5DYXfh5h963CgYcKB+6mdlpKiqhCRjFPTCkhBJcSAAYA066/SKjyavlP6t211UnjPiMOuPr68P77+IytVeA7UM/YZcIbgNtCCMwUO/4C2/JtqKy+k7EjyCLPudDN9oGwAm9WzauPfab23NrBH1DIab0ajIHJvJce8yZsoyLjEpbPfdeJyAw/z1kSPsPqrW2j125/Lt9W67JMXvuTpzb7Yy0zSqjMGTmaIjAQm20jU9ZQZgFt5C8CiXT082cReihtl8lY5ys2v1rWyU9AVpaJ0JgK42layaX1ib2YC6WVUmqDmWZRS+0egE1XKBsxrBOmQPCO7ejiiChU1BFD+gtnBpC2LxXZTxWRmSog0DpYrbenCxEp6uCtRkcupdGm7CygID6WrNqR51RL9BPxZjIlhFXQPriSJcw5fWAylha+3XZ70x6QxKMNpV2JPCJJIlCadjgwGaA5bB1gDadCoog6eWMd3/kuUXSayrwL7XlCdxSTqKtmjJELXrqs56gA3hDsJ6zB5/qd7BVa8/FVgDN8jcpmxu4US+hTX/AV3LAnJKiFODbJyo8nZ8klFvKeJRaCbQLpTaNPPrh86ohsx8W+vmGAKREJzdKyBq2R+Vtn0iS3RWOwOcrWCuEbQHUl2Qqu1hF3n3lQF2jEJ4/cCNPAWp/vbBEAQx8wiGmL8tvmuY+8P974tWlOt32moLHcDoGQ8fTdjuEbUJrmBaPJt8yAf10orT0s84ghmrvKzcWYGcpcn/9oI5+mGc2Cq+fa7HuahdTP/9ei/3vicA3fMxWcgM32Ree7K88cVp7nLiP5V5nsPM3/AvpkLLBsaIh91PB2YfK2Ml5aAKP4OZBBlh81lBWJ15jhJ1L0/SSiUKszU68NSC21MCURl6ie54thW1qWPvQKI7E893NTp6QZInPkYLPf3RmmmKlIL0whcalQKUH06oMRpAvbLCmvERIWkThGKaPqlshSEdzCQQUHFri2jhM0pZrd0vdo7msgEM1FGrtTUxnRTKEoh1ULufSdBBdGW2LyF8wiKmWBYsEAe9QwUMG4I2xAaLkJ5oCQc4iZC64drZTC6R4jw933htGOceO57sHna4tSHtxAM9hTxqhEHWJ2/xZUvkQJKkLZMgv1QOULWAAGJBgekEUkTQWmugpZkE4JBQ4oA7GkEHoLq7JxcW4VWlmk3k3RBvR0AlvIyUtCZvvj352ucMi2P2D0OmlnjpEzVP9vNss+wj4KwYXtCiSEHRt+1FhgxGrlOywvic4kgEstVs1ujYn4hxJqTkeuZRedB45S4X55XvcNIjgQG3pPgr/OKd1BWP3uAPrJr1coUUijPrMAqUeN5a4OmmvaMz/sCHY5un109U8AAAD//4AUZiI=" + return "eJzsW0tv3DgSvvtXFHwMNt67Dwv0OsnAwCQOHOcs0FK1zDFb1JBUG51fP6DeD5Jiuym1g3GfZiylvo/FqmI9qI/wjIdrIL8KgRcAiiqG13C50f9/eQEgkCGReA2PqMgFQIIyFjRXlGfX8L8LAIDyXfjKk4JpEVuKLJHX5aOPkJEdduL1Tx1yvIZU8CKv/2KQORTTFyWLx/btiCbt80bwMx5euOj/3Si++lXU+yLh9tMEMuZCICNBEG86WSYohRnJ1MkolRgTgEDJCxHjRH5/Q2ak309ljHerDzlYjGtBM7B96P7S+ljjVYSCm8ptEHPB9zRBsQColvFfjSJzMtB2h25+uip8OORvY2lt+CjUExf0V+WCooozgUA3fdkwkN3Cx4ruqTownspZvxkHzI7GlhRMRaWnXMOWMIl+7val9C3YclHHq4YPaEJX3l6ImaLqYNScyW1m9HZrlmei0KcRM0J3MqIZVZQoTKLHQ1TIifu4qXnQ07+bEgtaLHg8gAXLRhvcxj6kara/IwiDyQnGNFK6x2wdLn84oboTWaxD54cDqCGzLRhbh80XF1Krm/gJd2QF1Zhxhl539cHqZfzxL4yV4XH1IJqj2nst2pE8p1la/5vLD5eneK91SYMzYYnosZkB8AkZMub5Go5hhekfYcZVBGayseM0VFCnS9kkYxmSse3bEVQ+u3Fc29enKzjDiEhJ02yHmYpcWwqeyjxiFfp3zxlCR8FpVSPiCW7L429cuazFusMfZ+w2yuuTPNoEzqLJ3v57aDIXNItpTtjqZL83yMfR1EzORXSC3dDjOYqq5ghb6Nw1cqc5Za9O19WCQS2nlXYFU/b11qDa0IjqOjbhkM2S2yyFKEy5MJcor4K9MUnsDiLtzOFBP2u5buhcaNNSFMf5zavLse82iXMFmUSxpzFGAv8uUFqC25z7ebjdjwoH7iscuJ3KaSkpogoZxTwxhYQQXEoAGABMq/4iocqr5D+qdtdSJ4X7THPAVdeHt9+HJ2ylAt+CesIuEl4B3BRCYKbY4T+wKVdDZfVOxg4gizznQhfbe8IKvFo1rj70mdpjawe/RyGn+WowBibxXv2YV2EbOzKuxvKpa500meHtnCXhI6ze2tZ77fLn4m2ll2Xi2p88tckf9zKjhMqckYPJAwOx2TRtyhrK3EAb2YtAIh31/EmE7kvZZTDW8YrNa8s6+QnIyjIRGlNhPE2rdml9Yi9mQmnVKbXBTKOoJXcPwKZLlI0Y1glTIHjHdnR+RBQq6vAh/cKJDqTlS0V2047ITBYQSA9W6e3pQkSK2nmr0ZGr02jr7CzQQXwoWbUjz2kv0a+JNxMpIWwH7ZMrWMKc0QcmYynh2223F+0BSTzYUFpN5BFJEoHStMOByQDNYeMAazgVEkXUtTfWsZ2fEkXXU5k3oR1P6JZiEnXZjNFzwasv6zkqgFc4+xE6+FqvyZ6hNT/fDnCGL1FZzNiNYon+1Dd8ATfsEQFqIY5NsPLjyVlyDkXescRCsA0gvWn00QeXTx6RbbnY1TcMMCUioVla5qA1Mn/tTJrkNm8MNkfZWCF8HajOJNuGq3XE3Wce1ASa5pNHbISpY63PdzYJgKENGJppi/Lb5LlPe3+88WvTnG77TEJjuR0CIf3ppx3D16E0zTN6k2+aAb+dK609LPPwIZq70s3FmBnSXJ//aNtvjCh95r2VK3MNn7d3Ze591vg+a3yfNXpqw4Mc/I6zRm2rNAs+bNRiP9Is5LjxfXT375ul1fA9UcEJ2GSf9YhaeUS78pG0zIS0imSnjUjPMM6JBZb9H2KfzL5+jnNTCS8lgHFWNOgaLz+ZLQs4r6nsGxqA+nXUQ2lnprwdkFpKMSUR1wwyzxfDtnR4+tAr3CDIcz8zdU6AQkSOHGzyu08sqE7xtWIKiUu5SgmitQ9GkM5ts6S8dU1YROIYpYyqRHcpD27hoIIDC1ybxwmaUs1u6VLgrgYC0dQCdqOmMqKZQlHO9hcy6VsJLow2xeTPmEVUygLFgg72oGGggnF72IDQchc+BoScdz5ywbWhlZNDusPI8KlQw2jLuPFc9+DzvUUpD26gGewoY1SidjG7fQsqn6MEFaFsGUXdU/kMFoABCYZ7ZBFJU4GpzkIWpFNCgQPKQCwphN7CKm1cnFuFViapt1O0AT0dwBYy8pKQWf74M/0VDtn2e2+vk3bmGDlhSPrDJtlnDopCcGHr4kDYWxafNRYYsdppB5Z9rpkAcC5l1ezWuED0qYSaG7vVbRcdBw5S4W55XncNIjgQG3qPgr/MDQaDsPq/A+iN30ZTopDG/swCpB40ljs7aDrNM9/BBevvto8u/gkAAP//K/KugQ==" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml new file mode 100644 index 00000000000..1b2cefd8194 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -0,0 +1,45 @@ +- name: platformlogs + type: group + release: beta + default_field: false + description: > + Fields for Azure platform logs. + fields: + - name: identity + type: group + description: > + Identity + - name: operation_name + type: keyword + description: > + Operation name + - name: result_type + type: keyword + description: > + Result type + - name: result_signature + type: keyword + description: > + Result signature + - name: category + type: keyword + description: > + Category + - name: event_category + type: keyword + description: > + Event Category + - name: properties + type: group + description: > + Properties + fields: + - name: service_request_id + type: keyword + description: > + Service Request Id + - name: status_code + type: keyword + description: > + Status code + diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml new file mode 100644 index 00000000000..77c7ea3f5d0 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml @@ -0,0 +1,16 @@ +type: azure-eventhub +connection_string: {{ .connection_string }} +eventhub: {{ .eventhub }} +consumer_group: {{ .consumer_group }} +storage_account: {{ .storage_account }} +storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} +storage_account_container: filebeat-activitylogs-{{ .eventhub }} +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/config/file.yml b/x-pack/filebeat/module/azure/platformlogs/config/file.yml new file mode 100644 index 00000000000..e9470671e07 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/config/file.yml @@ -0,0 +1,14 @@ +type: log +paths: + {{ range $i, $path := .paths }} +- {{$path}} + {{ end }} +exclude_files: [".gz$"] +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml new file mode 100644 index 00000000000..c54f6d98f0e --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -0,0 +1,165 @@ +description: Pipeline for parsing azure platform logs. +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: azure + target_field: azure-eventhub + ignore_missing: true +- script: + source: ctx.message = ctx.message.replace(params.empty_field_name, '') + params: + empty_field_name: '"":"",' + ignore_failure: true +- json: + field: message + target_field: azure.platformlogs +- date: + field: azure.platformlogs.time + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 +- remove: + field: + - message + - azure.platformlogs.time + ignore_missing: true +- rename: + field: azure.platformlogs.resourceId + target_field: azure.resource_id + ignore_missing: true +- rename: + field: azure.platformlogs.callerIpAddress + target_field: source.ip + ignore_missing: true +- rename: + field: azure.platformlogs.level + target_field: log.level + ignore_missing: true +- rename: + field: azure.platformlogs.durationMs + target_field: event.duration + ignore_missing: true +- script: + lang: painless + source: if (ctx.event.duration!= null) {ctx.event.duration = ctx.event.duration + * params.param_nano;} + params: + param_nano: 1000000 + ignore_failure: true +- rename: + field: azure.platformlogs.location + target_field: geo.name + ignore_missing: true +- script: + lang: painless + source: >- + if (ctx?.azure?.platformlogs?.properties?.eventCategory != null) { + ctx.azure.platformlogs.event_category = ctx.azure.platformlogs.properties.eventCategory; + } + else if (ctx?.azure?.platformlogs?.properties?.policies != null) { + ctx.azure.platformlogs.event_category = 'Policy'; + } + else { + ctx.azure.platformlogs.event_category = 'Administrative'; + } + ignore_failure: true +- rename: + field: azure.platformlogs.resultType + target_field: azure.platformlogs.result_type + ignore_missing: true +- convert: + field: azure.platformlogs.result_type + target_field: event.outcome + type: string + if: "ctx?.azure?.platformlogs?.result_type != null && ctx.azure.platformlogs.result_type instanceof String && (ctx.azure.platformlogs.result_type.toLowerCase() == 'success' || ctx.azure.platformlogs.result_type.toLowerCase() == 'failure')" +- convert: + field: azure.platformlogs.properties.result + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)" +- rename: + field: azure.platformlogs.operationName + target_field: azure.platformlogs.operation_name + ignore_missing: true +- convert: + field: azure.platformlogs.operation_name + target_field: event.action + type: string + ignore_missing: true +- rename: + field: azure.platformlogs.resultSignature + target_field: azure.platformlogs.result_signature + ignore_missing: true +- rename: + field: azure.platformlogs.correlationId + target_field: azure.correlation_id + ignore_missing: true +- rename: + field: azure.platformlogs.properties.serviceRequestId + target_field: azure.platformlogs.properties.service_request_id + ignore_missing: true +- rename: + field: azure.platformlogs.properties.statusMessage + target_field: message + ignore_missing: true +- rename: + field: azure.platformlogs.properties.statusCode + target_field: azure.platformlogs.properties.status_code + ignore_missing: true +- geoip: + field: source.ip + target_field: geo + ignore_missing: true +- script: + lang: painless + ignore_failure: true + params: + "write": + type: + - change + "read": + type: + - access + "delete": + type: + - deletion + "action": + type: + - change + source: >- + if (ctx?.azure?.platformlogs?.category == null) { + return; + } + def hm = new HashMap(params.get(ctx.azure.platformlogs.category.toLowerCase())); + hm.forEach((k, v) -> ctx.event[k] = v); +- geoip: + field: source.ip + target_field: source.geo +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- set: + field: event.kind + value: event +- pipeline: + name: '{< IngestPipeline "azure-shared-pipeline" >}' +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/azure/platformlogs/manifest.yml b/x-pack/filebeat/module/azure/platformlogs/manifest.yml new file mode 100644 index 00000000000..92c9682ebaa --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/manifest.yml @@ -0,0 +1,20 @@ +module_version: 1.0 + +var: + - name: input + default: azure-eventhub + - name: eventhub + default: "insights-operational-logs" + - name: consumer_group + default: "$Default" + - name: connection_string + - name: storage_account + - name: storage_account_key + - name: resource_manager_endpoint + - name: tags + default: [forwarded] + +ingest_pipeline: + - ingest/pipeline.yml + - ../azure-shared-pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log new file mode 100644 index 00000000000..4b47c46d236 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log @@ -0,0 +1 @@ +{"callerIpAddress":"51.251.141.41","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json new file mode 100644 index 00000000000..3f86faee084 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json @@ -0,0 +1,67 @@ +[ + { + "@timestamp": "2019-10-24T00:13:46.355Z", + "azure.activitylogs.category": "Action", + "azure.activitylogs.event_category": "Administrative", + "azure.activitylogs.identity.authorization.action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", + "azure.activitylogs.identity.authorization.evidence.principal_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.authorization.evidence.principal_type": "ServicePrincipal", + "azure.activitylogs.identity.authorization.evidence.role": "Azure EventGrid Service BuiltIn Role", + "azure.activitylogs.identity.authorization.evidence.role_assignment_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.authorization.evidence.role_assignment_scope": "/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.authorization.evidence.role_definition_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.authorization.scope": "/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", + "azure.activitylogs.identity.claims.aio": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.appid": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.appidacr": "2", + "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/", + "azure.activitylogs.identity.claims.exp": "1571904826", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/identityprovider": "https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.iat": "1571875726", + "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/", + "azure.activitylogs.identity.claims.nbf": "1571875726", + "azure.activitylogs.identity.claims.uti": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.activitylogs.identity.claims.ver": "1.0", + "azure.activitylogs.operation_name": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "azure.activitylogs.result_signature": "Started.", + "azure.activitylogs.result_type": "Start", + "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "azure.resource.authorization_rule": "ROOTMANAGESHAREDACCESSKEY", + "azure.resource.group": "SA-HEMA", + "azure.resource.id": "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", + "azure.resource.namespace": "AZURELSEVENTS", + "azure.resource.provider": "MICROSOFT.EVENTHUB", + "azure.subscription_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", + "cloud.provider": "azure", + "event.action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", + "event.dataset": "azure.activitylogs", + "event.duration": 0, + "event.kind": "event", + "event.module": "azure", + "event.type": [ + "change" + ], + "fileset.name": "activitylogs", + "geo.continent_name": "Europe", + "geo.country_iso_code": "GB", + "geo.country_name": "United Kingdom", + "geo.location.lat": 51.4964, + "geo.location.lon": -0.1224, + "input.type": "log", + "log.level": "Information", + "log.offset": 0, + "service.type": "azure", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "GB", + "source.geo.country_name": "United Kingdom", + "source.geo.location.lat": 51.4964, + "source.geo.location.lon": -0.1224, + "source.ip": "51.251.141.41", + "tags": [ + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log new file mode 100644 index 00000000000..d1f15fa5d1d --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log @@ -0,0 +1 @@ +{"time":"2015-01-21T22:14:26.9792776Z","resourceId":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","operationName":"microsoft.support/supporttickets/write","category":"Write","resultType":"Success","resultSignature":"Succeeded.Created","durationMs":2826,"callerIpAddress":"111.111.111.11","correlationId":"c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"role":"Subscription Admin"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"1e8d8218-c5e7-4578-9acc-9abbd5d23315 ","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"admin@contoso.com","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" admin@contoso.com","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}},"level":"Information","location":"global","properties":{"statusCode":"Created","serviceRequestId":"50d5cddb-8ca0-47ad-9b80-6cde2207f97c"}} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json new file mode 100644 index 00000000000..5f14108e4c4 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json @@ -0,0 +1,77 @@ +[ + { + "@timestamp": "2015-01-21T22:14:26.979Z", + "azure.activitylogs.category": "Write", + "azure.activitylogs.event_category": "Administrative", + "azure.activitylogs.identity.authorization.action": "microsoft.support/supporttickets/write", + "azure.activitylogs.identity.authorization.evidence.role": "Subscription Admin", + "azure.activitylogs.identity.authorization.scope": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", + "azure.activitylogs.identity.claims.appid": "c44b4083-3bq0-49c1-b47d-974e53cbdf3c", + "azure.activitylogs.identity.claims.appidacr": "2", + "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/", + "azure.activitylogs.identity.claims.exp": "1421880271", + "azure.activitylogs.identity.claims.groups": "cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnclassreference": "1", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "2468adf0-8211-44e3-95xq-85137af64708", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", + "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "1e8d8218-c5e7-4578-9acc-9abbd5d23315 ", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": " admin@contoso.com", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Smith", + "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "admin@contoso.com", + "azure.activitylogs.identity.claims.iat": "1421876371", + "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/", + "azure.activitylogs.identity.claims.nbf": "1421876371", + "azure.activitylogs.identity.claims.puid": "20030000801A118C", + "azure.activitylogs.identity.claims.ver": "1.0", + "azure.activitylogs.identity.claims_initiated_by_user.fullname": "John Smith", + "azure.activitylogs.identity.claims_initiated_by_user.givenname": "John", + "azure.activitylogs.identity.claims_initiated_by_user.name": " admin@contoso.com", + "azure.activitylogs.identity.claims_initiated_by_user.schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims", + "azure.activitylogs.identity.claims_initiated_by_user.surname": "Smith", + "azure.activitylogs.operation_name": "microsoft.support/supporttickets/write", + "azure.activitylogs.properties.service_request_id": "50d5cddb-8ca0-47ad-9b80-6cde2207f97c", + "azure.activitylogs.properties.status_code": "Created", + "azure.activitylogs.result_signature": "Succeeded.Created", + "azure.activitylogs.result_type": "Success", + "azure.correlation_id": "c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8", + "azure.resource.id": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", + "azure.resource.provider": "microsoft.support/supporttickets/115012112305841", + "cloud.provider": "azure", + "event.action": "microsoft.support/supporttickets/write", + "event.dataset": "azure.activitylogs", + "event.duration": -1468967296, + "event.kind": "event", + "event.module": "azure", + "event.outcome": "success", + "event.type": [ + "change" + ], + "fileset.name": "activitylogs", + "geo.continent_name": "Asia", + "geo.country_iso_code": "JP", + "geo.country_name": "Japan", + "geo.location.lat": 35.69, + "geo.location.lon": 139.69, + "input.type": "log", + "log.level": "Information", + "log.offset": 0, + "service.type": "azure", + "source.as.number": 2516, + "source.as.organization.name": "KDDI CORPORATION", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "JP", + "source.geo.country_name": "Japan", + "source.geo.location.lat": 35.69, + "source.geo.location.lon": 139.69, + "source.ip": "111.111.111.11", + "tags": [ + "forwarded" + ], + "user.domain": "contoso.com", + "user.full_name": "John Smith", + "user.name": "admin" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index 0c7eb3d6e01..da2618353c2 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -17,6 +17,16 @@ # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" + platformlogs: + enabled: false + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" + + auditlogs: enabled: false # var: From 70369e74df9571905cbd5a8a4f7b452af13dc560 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Nov 2020 11:29:24 +0100 Subject: [PATCH 03/19] fix --- filebeat/docs/fields.asciidoc | 90 +++++++++++++++++++ filebeat/docs/modules/azure.asciidoc | 13 +++ .../azure/activitylogs/ingest/pipeline.yml | 1 + .../azure/platformlogs/ingest/pipeline.yml | 1 + .../azure/platformlogs/test/platformlogs.log | 2 +- 5 files changed, 106 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index b1ee49fed5c..a6dbef5876d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3050,6 +3050,96 @@ type: keyword ip Address +type: keyword + +-- + +[float] +=== platformlogs + +Fields for Azure platform logs. + + + +[float] +=== identity + +Identity + + +*`azure.platformlogs.operation_name`*:: ++ +-- +Operation name + + +type: keyword + +-- + +*`azure.platformlogs.result_type`*:: ++ +-- +Result type + + +type: keyword + +-- + +*`azure.platformlogs.result_signature`*:: ++ +-- +Result signature + + +type: keyword + +-- + +*`azure.platformlogs.category`*:: ++ +-- +Category + + +type: keyword + +-- + +*`azure.platformlogs.event_category`*:: ++ +-- +Event Category + + +type: keyword + +-- + +[float] +=== properties + +Properties + + + +*`azure.platformlogs.properties.service_request_id`*:: ++ +-- +Service Request Id + + +type: keyword + +-- + +*`azure.platformlogs.properties.status_code`*:: ++ +-- +Status code + + type: keyword -- diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 853fba43756..5b52aba8f22 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -24,6 +24,9 @@ The module contains the following filesets: `activitylogs` :: Will retrieve azure activity logs. Control-plane events on Azure Resource Manager resources. Activity logs provide insight into the operations that were performed on resources in your subscription. +`platformlogs` :: +Will retrieve azure platform logs. Platform logs provide detailed diagnostic and auditing information for Azure resources and the Azure platform they depend on. + `signinlogs` :: Will retrieve azure Active Directory sign-in logs. The sign-ins report provides information about the usage of managed applications and user sign-in activities. @@ -45,6 +48,16 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi storage_account_key: "" resource_manager_endpoint: "" + platformlogs: + enabled: false + var: + eventhub: "" + consumer_group: "$Default" + connection_string: "" + storage_account: "" + storage_account_key: "" + resource_manager_endpoint: "" + auditlogs: enabled: false var: diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index 2d75cb07241..a7a581db2b2 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -201,6 +201,7 @@ processors: - geoip: field: source.ip target_field: source.geo + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb field: source.ip diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml index c54f6d98f0e..e0845835060 100644 --- a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -138,6 +138,7 @@ processors: - geoip: field: source.ip target_field: source.geo + ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb field: source.ip diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log index 4b47c46d236..80af31aa454 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log @@ -1 +1 @@ -{"callerIpAddress":"51.251.141.41","category":"Action","correlationId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","durationMs":0,"identity":{"authorization":{"action":"Microsoft.EventHub/namespaces/authorizationRules/listKeys/action","evidence":{"principalId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","principalType":"ServicePrincipal","role":"Azure EventGrid Service BuiltIn Role","roleAssignmentId":"8a4de8b5-095c-47d0-a96f-a75130c61d53","roleAssignmentScope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53","roleDefinitionId":"8a4de8b5-095c-47d0-a96f-a75130c61d53"},"scope":"/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey"},"claims":{"aio":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","appidacr":"2","aud":"https://management.core.windows.net/","exp":"1571904826","http://schemas.microsoft.com/identity/claims/identityprovider":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","http://schemas.microsoft.com/identity/claims/objectidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.microsoft.com/identity/claims/tenantid":"8a4de8b5-095c-47d0-a96f-a75130c61d53","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"8a4de8b5-095c-47d0-a96f-a75130c61d53","iat":"1571875726","iss":"https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/","nbf":"1571875726","uti":"8a4de8b5-095c-47d0-a96f-a75130c61d53","ver":"1.0"}},"level":"Information","location":"global","operationName":"MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION","resourceId":"/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY","resultSignature":"Started.","resultType":"Start","time":"2019-10-24T00:13:46.3554259Z"} +{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","SubscriptionId":"7657426d-c4c3-44ac-88a2-3b2cd59e6dba","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"} From fc7b82a6c6c4645b23a2e93db18add47711524a3 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 3 Nov 2020 12:10:29 +0100 Subject: [PATCH 04/19] separate pr --- x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml index a7a581db2b2..2d75cb07241 100644 --- a/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml @@ -201,7 +201,6 @@ processors: - geoip: field: source.ip target_field: source.geo - ignore_missing: true - geoip: database_file: GeoLite2-ASN.mmdb field: source.ip From 591aea61ab1ba90d7d5c62893d7c7db288b89d9d Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 4 Nov 2020 10:38:01 +0100 Subject: [PATCH 05/19] work on platform --- x-pack/filebeat/filebeat.reference.yml | 12 +-- x-pack/filebeat/module/azure/_meta/config.yml | 12 +-- .../platformlogs/config/azure-eventhub.yml | 2 +- .../module/azure/platformlogs/manifest.yml | 1 - .../test/supporttickets_write.log | 1 - .../supporttickets_write.log-expected.json | 77 ------------------- x-pack/filebeat/modules.d/azure.yml.disabled | 12 +-- 7 files changed, 19 insertions(+), 98 deletions(-) delete mode 100644 x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log delete mode 100644 x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index a47280871a8..a9848c135cc 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -367,12 +367,12 @@ filebeat.modules: platformlogs: enabled: false - # var: - # eventhub: "insights-logs-signinlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" auditlogs: diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 588cd3c3078..fa9001e5fbe 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -16,12 +16,12 @@ platformlogs: enabled: false - # var: - # eventhub: "insights-logs-signinlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" auditlogs: diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml index 77c7ea3f5d0..496480aa1d0 100644 --- a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml @@ -5,7 +5,7 @@ consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} resource_manager_endpoint: {{ .resource_manager_endpoint }} -storage_account_container: filebeat-activitylogs-{{ .eventhub }} +storage_account_container: filebeat-platformlogs-{{ .eventhub }} tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/azure/platformlogs/manifest.yml b/x-pack/filebeat/module/azure/platformlogs/manifest.yml index 92c9682ebaa..a67dc604dd2 100644 --- a/x-pack/filebeat/module/azure/platformlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/platformlogs/manifest.yml @@ -4,7 +4,6 @@ var: - name: input default: azure-eventhub - name: eventhub - default: "insights-operational-logs" - name: consumer_group default: "$Default" - name: connection_string diff --git a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log deleted file mode 100644 index d1f15fa5d1d..00000000000 --- a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log +++ /dev/null @@ -1 +0,0 @@ -{"time":"2015-01-21T22:14:26.9792776Z","resourceId":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","operationName":"microsoft.support/supporttickets/write","category":"Write","resultType":"Success","resultSignature":"Succeeded.Created","durationMs":2826,"callerIpAddress":"111.111.111.11","correlationId":"c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"role":"Subscription Admin"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"1e8d8218-c5e7-4578-9acc-9abbd5d23315 ","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"admin@contoso.com","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" admin@contoso.com","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}},"level":"Information","location":"global","properties":{"statusCode":"Created","serviceRequestId":"50d5cddb-8ca0-47ad-9b80-6cde2207f97c"}} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json deleted file mode 100644 index 5f14108e4c4..00000000000 --- a/x-pack/filebeat/module/azure/platformlogs/test/supporttickets_write.log-expected.json +++ /dev/null @@ -1,77 +0,0 @@ -[ - { - "@timestamp": "2015-01-21T22:14:26.979Z", - "azure.activitylogs.category": "Write", - "azure.activitylogs.event_category": "Administrative", - "azure.activitylogs.identity.authorization.action": "microsoft.support/supporttickets/write", - "azure.activitylogs.identity.authorization.evidence.role": "Subscription Admin", - "azure.activitylogs.identity.authorization.scope": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", - "azure.activitylogs.identity.claims.appid": "c44b4083-3bq0-49c1-b47d-974e53cbdf3c", - "azure.activitylogs.identity.claims.appidacr": "2", - "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/", - "azure.activitylogs.identity.claims.exp": "1421880271", - "azure.activitylogs.identity.claims.groups": "cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnclassreference": "1", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "2468adf0-8211-44e3-95xq-85137af64708", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/scope": "user_impersonation", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "1e8d8218-c5e7-4578-9acc-9abbd5d23315 ", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": " admin@contoso.com", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Smith", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "admin@contoso.com", - "azure.activitylogs.identity.claims.iat": "1421876371", - "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/", - "azure.activitylogs.identity.claims.nbf": "1421876371", - "azure.activitylogs.identity.claims.puid": "20030000801A118C", - "azure.activitylogs.identity.claims.ver": "1.0", - "azure.activitylogs.identity.claims_initiated_by_user.fullname": "John Smith", - "azure.activitylogs.identity.claims_initiated_by_user.givenname": "John", - "azure.activitylogs.identity.claims_initiated_by_user.name": " admin@contoso.com", - "azure.activitylogs.identity.claims_initiated_by_user.schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims", - "azure.activitylogs.identity.claims_initiated_by_user.surname": "Smith", - "azure.activitylogs.operation_name": "microsoft.support/supporttickets/write", - "azure.activitylogs.properties.service_request_id": "50d5cddb-8ca0-47ad-9b80-6cde2207f97c", - "azure.activitylogs.properties.status_code": "Created", - "azure.activitylogs.result_signature": "Succeeded.Created", - "azure.activitylogs.result_type": "Success", - "azure.correlation_id": "c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8", - "azure.resource.id": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841", - "azure.resource.provider": "microsoft.support/supporttickets/115012112305841", - "cloud.provider": "azure", - "event.action": "microsoft.support/supporttickets/write", - "event.dataset": "azure.activitylogs", - "event.duration": -1468967296, - "event.kind": "event", - "event.module": "azure", - "event.outcome": "success", - "event.type": [ - "change" - ], - "fileset.name": "activitylogs", - "geo.continent_name": "Asia", - "geo.country_iso_code": "JP", - "geo.country_name": "Japan", - "geo.location.lat": 35.69, - "geo.location.lon": 139.69, - "input.type": "log", - "log.level": "Information", - "log.offset": 0, - "service.type": "azure", - "source.as.number": 2516, - "source.as.organization.name": "KDDI CORPORATION", - "source.geo.continent_name": "Asia", - "source.geo.country_iso_code": "JP", - "source.geo.country_name": "Japan", - "source.geo.location.lat": 35.69, - "source.geo.location.lon": 139.69, - "source.ip": "111.111.111.11", - "tags": [ - "forwarded" - ], - "user.domain": "contoso.com", - "user.full_name": "John Smith", - "user.name": "admin" - } -] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index da2618353c2..ae50c6abaca 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -19,12 +19,12 @@ platformlogs: enabled: false - # var: - # eventhub: "insights-logs-signinlogs" - # consumer_group: "$Default" - # connection_string: "" - # storage_account: "" - # storage_account_key: "" + # var: + # eventhub: "insights-logs-signinlogs" + # consumer_group: "$Default" + # connection_string: "" + # storage_account: "" + # storage_account_key: "" auditlogs: From b479600c0eefd53f57f31c53e3aee5c0bad17516 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 4 Nov 2020 14:35:12 +0100 Subject: [PATCH 06/19] work --- .../module/azure/platformlogs/ingest/pipeline.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml index e0845835060..c600061d2a6 100644 --- a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -97,14 +97,6 @@ processors: field: azure.platformlogs.correlationId target_field: azure.correlation_id ignore_missing: true -- rename: - field: azure.platformlogs.properties.serviceRequestId - target_field: azure.platformlogs.properties.service_request_id - ignore_missing: true -- rename: - field: azure.platformlogs.properties.statusMessage - target_field: message - ignore_missing: true - rename: field: azure.platformlogs.properties.statusCode target_field: azure.platformlogs.properties.status_code From cabc83944b399474457daf50dbd3c953050033d7 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 5 Nov 2020 14:01:52 +0100 Subject: [PATCH 07/19] platforms --- filebeat/docs/fields.asciidoc | 10 -- x-pack/filebeat/filebeat.reference.yml | 2 +- x-pack/filebeat/module/azure/_meta/config.yml | 2 +- x-pack/filebeat/module/azure/fields.go | 2 +- .../azure/platformlogs/_meta/fields.yml | 5 +- .../azure/platformlogs/ingest/pipeline.yml | 33 +++++++ .../test/platformlogs.log-expected.json | 97 ++++++++----------- x-pack/filebeat/modules.d/azure.yml.disabled | 2 +- 8 files changed, 76 insertions(+), 77 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index a6dbef5876d..1a61323d2e9 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3124,16 +3124,6 @@ Properties -*`azure.platformlogs.properties.service_request_id`*:: -+ --- -Service Request Id - - -type: keyword - --- - *`azure.platformlogs.properties.status_code`*:: + -- diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index deebf8bd5a4..f32b26a7311 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -368,7 +368,7 @@ filebeat.modules: platformlogs: enabled: false # var: - # eventhub: "insights-logs-signinlogs" + # eventhub: "" # consumer_group: "$Default" # connection_string: "" # storage_account: "" diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index fa9001e5fbe..fdea9b1f252 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -17,7 +17,7 @@ platformlogs: enabled: false # var: - # eventhub: "insights-logs-signinlogs" + # eventhub: "" # consumer_group: "$Default" # connection_string: "" # storage_account: "" diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index f3358cf79e8..19b3815cf50 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW0tv3DgSvvtXFHwMNt67Dwv0OsnAwCQOHOcs0FK1zDFb1JBUG51fP6DeD5Jiuym1g3GfZiylvo/FqmI9qI/wjIdrIL8KgRcAiiqG13C50f9/eQEgkCGReA2PqMgFQIIyFjRXlGfX8L8LAIDyXfjKk4JpEVuKLJHX5aOPkJEdduL1Tx1yvIZU8CKv/2KQORTTFyWLx/btiCbt80bwMx5euOj/3Si++lXU+yLh9tMEMuZCICNBEG86WSYohRnJ1MkolRgTgEDJCxHjRH5/Q2ak309ljHerDzlYjGtBM7B96P7S+ljjVYSCm8ptEHPB9zRBsQColvFfjSJzMtB2h25+uip8OORvY2lt+CjUExf0V+WCooozgUA3fdkwkN3Cx4ruqTownspZvxkHzI7GlhRMRaWnXMOWMIl+7val9C3YclHHq4YPaEJX3l6ImaLqYNScyW1m9HZrlmei0KcRM0J3MqIZVZQoTKLHQ1TIifu4qXnQ07+bEgtaLHg8gAXLRhvcxj6kara/IwiDyQnGNFK6x2wdLn84oboTWaxD54cDqCGzLRhbh80XF1Krm/gJd2QF1Zhxhl539cHqZfzxL4yV4XH1IJqj2nst2pE8p1la/5vLD5eneK91SYMzYYnosZkB8AkZMub5Go5hhekfYcZVBGayseM0VFCnS9kkYxmSse3bEVQ+u3Fc29enKzjDiEhJ02yHmYpcWwqeyjxiFfp3zxlCR8FpVSPiCW7L429cuazFusMfZ+w2yuuTPNoEzqLJ3v57aDIXNItpTtjqZL83yMfR1EzORXSC3dDjOYqq5ghb6Nw1cqc5Za9O19WCQS2nlXYFU/b11qDa0IjqOjbhkM2S2yyFKEy5MJcor4K9MUnsDiLtzOFBP2u5buhcaNNSFMf5zavLse82iXMFmUSxpzFGAv8uUFqC25z7ebjdjwoH7iscuJ3KaSkpogoZxTwxhYQQXEoAGABMq/4iocqr5D+qdtdSJ4X7THPAVdeHt9+HJ2ylAt+CesIuEl4B3BRCYKbY4T+wKVdDZfVOxg4gizznQhfbe8IKvFo1rj70mdpjawe/RyGn+WowBibxXv2YV2EbOzKuxvKpa500meHtnCXhI6ze2tZ77fLn4m2ll2Xi2p88tckf9zKjhMqckYPJAwOx2TRtyhrK3EAb2YtAIh31/EmE7kvZZTDW8YrNa8s6+QnIyjIRGlNhPE2rdml9Yi9mQmnVKbXBTKOoJXcPwKZLlI0Y1glTIHjHdnR+RBQq6vAh/cKJDqTlS0V2047ITBYQSA9W6e3pQkSK2nmr0ZGr02jr7CzQQXwoWbUjz2kv0a+JNxMpIWwH7ZMrWMKc0QcmYynh2223F+0BSTzYUFpN5BFJEoHStMOByQDNYeMAazgVEkXUtTfWsZ2fEkXXU5k3oR1P6JZiEnXZjNFzwasv6zkqgFc4+xE6+FqvyZ6hNT/fDnCGL1FZzNiNYon+1Dd8ATfsEQFqIY5NsPLjyVlyDkXescRCsA0gvWn00QeXTx6RbbnY1TcMMCUioVla5qA1Mn/tTJrkNm8MNkfZWCF8HajOJNuGq3XE3Wce1ASa5pNHbISpY63PdzYJgKENGJppi/Lb5LlPe3+88WvTnG77TEJjuR0CIf3ppx3D16E0zTN6k2+aAb+dK609LPPwIZq70s3FmBnSXJ//aNtvjCh95r2VK3MNn7d3Ze591vg+a3yfNXpqw4Mc/I6zRm2rNAs+bNRiP9Is5LjxfXT375ul1fA9UcEJ2GSf9YhaeUS78pG0zIS0imSnjUjPMM6JBZb9H2KfzL5+jnNTCS8lgHFWNOgaLz+ZLQs4r6nsGxqA+nXUQ2lnprwdkFpKMSUR1wwyzxfDtnR4+tAr3CDIcz8zdU6AQkSOHGzyu08sqE7xtWIKiUu5SgmitQ9GkM5ts6S8dU1YROIYpYyqRHcpD27hoIIDC1ybxwmaUs1u6VLgrgYC0dQCdqOmMqKZQlHO9hcy6VsJLow2xeTPmEVUygLFgg72oGGggnF72IDQchc+BoScdz5ywbWhlZNDusPI8KlQw2jLuPFc9+DzvUUpD26gGewoY1SidjG7fQsqn6MEFaFsGUXdU/kMFoABCYZ7ZBFJU4GpzkIWpFNCgQPKQCwphN7CKm1cnFuFViapt1O0AT0dwBYy8pKQWf74M/0VDtn2e2+vk3bmGDlhSPrDJtlnDopCcGHr4kDYWxafNRYYsdppB5Z9rpkAcC5l1ezWuED0qYSaG7vVbRcdBw5S4W55XncNIjgQG3qPgr/MDQaDsPq/A+iN30ZTopDG/swCpB40ljs7aDrNM9/BBevvto8u/gkAAP//K/KugQ==" + return "eJzsW0tv3DYQvvtXDHwMGvfuQ4GtkxQGmjhwnLNAS7Mya66oktQam19fUO8HSXG9lNZFsqfWUub7OJwZzoN6D894uAbyoxB4AaCoYngNlxv9/5cXAAIZEonX8IiKXAAkKGNBc0V5dg1/XAAAlO/CZ54UTIvYUmSJvC4fvYeM7LATr3/qkOM1pIIXef0Xg8yhmL4oWTy2b0c0aZ83gp/x8MJF/+9G8dWvot4XCbcfJpAxFwIZCYJ408kyQSnMSKZORqnEmAAESl6IGCfy+xsyI/1+KmO8W33IwWJcC5qB7UP3l9bHGq8iFNxUboOYC76nCYoFQLWM3zWKzMlA2x26+emq8OGQv4ylteGjUE9c0B+VC4oqzgQC3fRlw0B2Cx8ruqfqwHgqZ/1mHDA7GltSMBWVnnINW8Ik+rnbp9K3YMtFHa8aPqAJXXl7IWaKqoNRcya3mdHbrVmeiUKfRswI3cmIZlRRojCJHg9RISfu46bmQU//bkosaLHg8QAWLBttcBv7kKrZ/o4gDCYnGNNI6R6zdbj85YTqTmSxDp1vDqCGzLZgbB02n1xIrW7iJ9yRFVRjxhl63dU7q5fxx38wVobH1YNojmrvtWhH8pxmaf1vLt9dnuK91iUNzoQlosdmBsAnZMiY52s4hhWmf4QZVxGYycaO01BBnS5lk4xlSMa2b0dQ+ejGcW1fn67gDCMiJU2zHWYqcm0peCrziFXo3z1nCB0Fp1WNiCe4LY+/ceWyFusOf5yx2yivT/JoEziLJnv776HJXNAspjlhq5P92iAfR1MzORfRCXZDj+coqpojbKFz18id5pS9Ol1XCwa1nFbaFUzZ11uDakMjquvYhEM2S26zFKIw5cJcorwK9sYksTuItDOHB/2o5bqhc6FNS1Ec5zevLse+2iTOFWQSxZ7GGAn8t0BpCW5z7ufhdt8qHLivcOB2KqelpIgqZBTzxBQSQnApAWAAMK36i4Qqr5L/qNpdS50U7jPNAVddH95+H56wlQp8C+oJu0h4BXBTCIGZYoffYFOuhsrqnYwdQBZ5zoUutveEFXi1alx96DO1x9YOfo9CTvPVYAxM4r36Ma/CNnZkXI3lU9c6aTLD2zlLwkdYvbWt99rlz8XbSi/LxLW/eWqTP+5lRgmVOSMHkwcGYrNp2pQ1lLmBNrIXgUQ66vmTCN2XsstgrOMVm9eWdfITkJVlIjSmwniaVu3S+sRezITSqlNqg5lGUUvuHoBNlygbMawTpkDwju3o/IgoVNThQ/qFEx1Iy5eK7KYdkZksIJAerNLb04WIFLXzVqMjV6fR1tlZoIP4ULJqR57TXqJfE28mUkLYDtoHV7CEOaMPTMZSwrfbbi/aA5J4sKG0msgjkiQCpWmHA5MBmsPGAdZwKiSKqGtvrGM73yWKrqcyb0I7ntAtxSTqshmj54JXX9ZzVACvcPYjdPC5XpM9Q2t+vh3gDF+ispixG8US/akv+AJu2CMC1EIcm2Dlx5Oz5ByKvGOJhWAbQHrT6KMPLp88IttysatvGGBKREKztMxBa2T+2pk0yW3eGGyOsrFC+DpQnUm2DVfriLvPPKgJNM0nj9gIU8dan+9sEgBDGzA00xblt8lzn/b+eOPXpjnd9pmExnI7BEL603c7hq9DaZpn9CbfNAP+d6609rDMw4do7ko3F2NmSHN9/qNtvzGi9Jn3Vq7MNXze3pW5X7PGX7PGn2bWeNbB3iRIaQuhWfARnxb7nmYhh3y/BmY/3wSrhu+JCk7AJvusB8PKg9GVD4Jl5pJVJDttMHmGIUossOy6EPs89PXTk5tKeCkBjBOaQa92+XloWTZ5zULf0NjRr48dSjszReWA1FKKKYm4Jn95vhi2pa/Sh15hbp/nfmbqnLuEiBw52OR3HzZQnVhrxRQSl3KVEkRrH4wgndtmSXnXmbCIxDFKGVUZ71Ie3MJBBQcWuDaPEzSlmt3Sl/3uaiAQzW0/u1FTGdFMoSgn6guZ9K0EF0abYvJnzCIqZYFiQQd70DBQwbg9bEBouWsWA0LOmxa54NrQynkd3WFk+ECnYbRl3Hiue/D52qKUBzfQDHaUMSpRu5jdvgWVz1GCilC2jKLuqXwGC8CABMM9soikqcBUZyEL0imhwAFlIJYUQm9hlTYuzq1CK5PU2ynagJ4OYAsZeUnILH/8cfwKh2z7lbXXSTtzjJwwmvxmk+wzfUQhuLC1cyDs3YaPGguMWO2MAcub7DMB4FzKqtmtcW3nQwk1N+yq2y46Dhykwt3yvO4aRHAgNvQeBX+ZG8cFYfWnA+iN3wFTopDG/swCpB40ljs7aL4lmfn6LFijt3108V8AAAD//wuEi5I=" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index 1b2cefd8194..d4326bacd9d 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -34,12 +34,9 @@ description: > Properties fields: - - name: service_request_id - type: keyword - description: > - Service Request Id - name: status_code type: keyword description: > Status code + diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml index c600061d2a6..5bc67bdc336 100644 --- a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -21,6 +21,13 @@ processors: ignore_failure: true formats: - ISO8601 +- date: + field: azure.platformlogs.EventTimeString + target_field: '@timestamp' + ignore_failure: true + formats: + - ISO8601 + - "M/d/yyyy h:mm:ss a XXX" - remove: field: - message @@ -30,6 +37,27 @@ processors: field: azure.platformlogs.resourceId target_field: azure.resource_id ignore_missing: true +- rename: + field: azure.platformlogs.Region + target_field: cloud.region + ignore_missing: true +- json: + field: azure.platformlogs.EventProperties + target_field: azure.platformlogs.properties + ignore_failure: true +- remove: + if: ctx.azure.platformlogs.properties != null + field: + - azure.platformlogs.EventProperties + ignore_missing: true +- rename: + field: azure.platformlogs.EventName + target_field: event.action + ignore_missing: true +- rename: + field: azure.platformlogs.properties.log + target_field: message + ignore_missing: true - rename: field: azure.platformlogs.callerIpAddress target_field: source.ip @@ -80,6 +108,11 @@ processors: target_field: event.outcome type: string if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.properties?.result != null && ctx?.azure?.platformlogs?.properties?.result instanceof String && ['success', 'failure', 'unknown'].contains(ctx.azure?.platformlogs?.properties?.result)" +- convert: + field: azure.platformlogs.Status + target_field: event.outcome + type: string + if: "ctx?.event?.outcome == null && ctx?.azure?.platformlogs?.Status != null && ctx?.azure?.platformlogs?.Status instanceof String && ['success', 'failure', 'unknown', 'Succeeded', 'Failed'].contains(ctx.azure?.platformlogs?.Status)" - rename: field: azure.platformlogs.operationName target_field: azure.platformlogs.operation_name diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json index 3f86faee084..5455bd306a2 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json @@ -1,67 +1,46 @@ [ { - "@timestamp": "2019-10-24T00:13:46.355Z", - "azure.activitylogs.category": "Action", - "azure.activitylogs.event_category": "Administrative", - "azure.activitylogs.identity.authorization.action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action", - "azure.activitylogs.identity.authorization.evidence.principal_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.authorization.evidence.principal_type": "ServicePrincipal", - "azure.activitylogs.identity.authorization.evidence.role": "Azure EventGrid Service BuiltIn Role", - "azure.activitylogs.identity.authorization.evidence.role_assignment_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.authorization.evidence.role_assignment_scope": "/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.authorization.evidence.role_definition_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.authorization.scope": "/subscriptions/8a4de8b5-095c-47d0-a96f-a75130c61d53/resourceGroups/sa-hem/providers/Microsoft.EventHub/namespaces/azurelsevents/authorizationRules/RootManageSharedAccessKey", - "azure.activitylogs.identity.claims.aio": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.appid": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.appidacr": "2", - "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/", - "azure.activitylogs.identity.claims.exp": "1571904826", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/identityprovider": "https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.iat": "1571875726", - "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/8a4de8b5-095c-47d0-a96f-a75130c61d53/", - "azure.activitylogs.identity.claims.nbf": "1571875726", - "azure.activitylogs.identity.claims.uti": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.activitylogs.identity.claims.ver": "1.0", - "azure.activitylogs.operation_name": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "azure.activitylogs.result_signature": "Started.", - "azure.activitylogs.result_type": "Start", - "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "azure.resource.authorization_rule": "ROOTMANAGESHAREDACCESSKEY", - "azure.resource.group": "SA-HEMA", - "azure.resource.id": "/SUBSCRIPTIONS/8a4de8b5-095c-47d0-a96f-a75130c61d53/RESOURCEGROUPS/SA-HEMA/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/AZURELSEVENTS/AUTHORIZATIONRULES/ROOTMANAGESHAREDACCESSKEY", - "azure.resource.namespace": "AZURELSEVENTS", - "azure.resource.provider": "MICROSOFT.EVENTHUB", - "azure.subscription_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53", - "cloud.provider": "azure", - "event.action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION", - "event.dataset": "azure.activitylogs", - "event.duration": 0, - "event.kind": "event", - "event.module": "azure", - "event.type": [ - "change" - ], - "fileset.name": "activitylogs", - "geo.continent_name": "Europe", - "geo.country_iso_code": "GB", - "geo.country_name": "United Kingdom", - "geo.location.lat": 51.4964, - "geo.location.lon": -0.1224, + "cloud" : { + "region" : "West Europe", + "provider" : "azure" + }, + "@timestamp" : "2020-11-03T09:06:42.000Z", + "event" : { + "action" : "Retreive ConsumerGroup", + "ingested" : "2020-11-05T12:51:52.507448200Z", + "kind" : "event", + "outcome" : "succeeded" + }, + "azure" : { + "subscription_id" : "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "resource" : { + "name" : "OBSTESTEVENTHUBS", + "id" : "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "provider" : "MICROSOFT.EVENTHUB/NAMESPACES", + "group" : "OBS-TEST" + }, + "platformlogs" : { + "Status" : "Succeeded", + "SubscriptionId" : "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "Caller" : "Portal", + "ActivityId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8", + "EventTimeString" : "11/3/2020 9:06:42 AM +00:00", + "Environment" : "PROD", + "category" : "OperationalLogs", + "event_category" : "Administrative", + "ScaleUnit" : "PROD-AM3-AZ501", + "properties" : { + "SubscriptionId" : "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "TrackingId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2", + "Namespace" : "obstesteventhubs", + "Via" : "sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04&$skip=0&$top=100" + } + } + }, "input.type": "log", - "log.level": "Information", - "log.offset": 0, "service.type": "azure", - "source.geo.continent_name": "Europe", - "source.geo.country_iso_code": "GB", - "source.geo.country_name": "United Kingdom", - "source.geo.location.lat": 51.4964, - "source.geo.location.lon": -0.1224, - "source.ip": "51.251.141.41", "tags": [ "forwarded" ] } -] \ No newline at end of file +] diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index ae50c6abaca..dcf5b1764d7 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -20,7 +20,7 @@ platformlogs: enabled: false # var: - # eventhub: "insights-logs-signinlogs" + # eventhub: "" # consumer_group: "$Default" # connection_string: "" # storage_account: "" From 7929b8392aa558cf423fb2845ba77a352c370a32 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 5 Nov 2020 14:03:44 +0100 Subject: [PATCH 08/19] changelog --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index eed469a7558..4ea1ff6bc19 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -656,6 +656,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update Okta documentation for new stateful restarts. {pull}22091[22091] - Copy tag names from MISP data into events. {pull}21664[21664] - Added TLS JA3 fingerprint, certificate not_before/not_after, certificate SHA1 hash, and certificate subject fields to Zeek SSL dataset. {pull}21696[21696] +- Add platform logs in the azure filebeat module. {pull}22371[22371] *Heartbeat* From faac020c7c596f5cefbb76fed508aa7cfd402f69 Mon Sep 17 00:00:00 2001 From: Mariana Date: Thu, 5 Nov 2020 15:31:24 +0100 Subject: [PATCH 09/19] fix file --- filebeat/docs/fields.asciidoc | 6 ------ x-pack/filebeat/module/azure/fields.go | 2 +- x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml | 4 ---- 3 files changed, 1 insertion(+), 11 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 1a61323d2e9..908422e17a2 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3061,12 +3061,6 @@ Fields for Azure platform logs. -[float] -=== identity - -Identity - - *`azure.platformlogs.operation_name`*:: + -- diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 19b3815cf50..76a904e0e30 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW0tv3DYQvvtXDHwMGvfuQ4GtkxQGmjhwnLNAS7Mya66oktQam19fUO8HSXG9lNZFsqfWUub7OJwZzoN6D894uAbyoxB4AaCoYngNlxv9/5cXAAIZEonX8IiKXAAkKGNBc0V5dg1/XAAAlO/CZ54UTIvYUmSJvC4fvYeM7LATr3/qkOM1pIIXef0Xg8yhmL4oWTy2b0c0aZ83gp/x8MJF/+9G8dWvot4XCbcfJpAxFwIZCYJ408kyQSnMSKZORqnEmAAESl6IGCfy+xsyI/1+KmO8W33IwWJcC5qB7UP3l9bHGq8iFNxUboOYC76nCYoFQLWM3zWKzMlA2x26+emq8OGQv4ylteGjUE9c0B+VC4oqzgQC3fRlw0B2Cx8ruqfqwHgqZ/1mHDA7GltSMBWVnnINW8Ik+rnbp9K3YMtFHa8aPqAJXXl7IWaKqoNRcya3mdHbrVmeiUKfRswI3cmIZlRRojCJHg9RISfu46bmQU//bkosaLHg8QAWLBttcBv7kKrZ/o4gDCYnGNNI6R6zdbj85YTqTmSxDp1vDqCGzLZgbB02n1xIrW7iJ9yRFVRjxhl63dU7q5fxx38wVobH1YNojmrvtWhH8pxmaf1vLt9dnuK91iUNzoQlosdmBsAnZMiY52s4hhWmf4QZVxGYycaO01BBnS5lk4xlSMa2b0dQ+ejGcW1fn67gDCMiJU2zHWYqcm0peCrziFXo3z1nCB0Fp1WNiCe4LY+/ceWyFusOf5yx2yivT/JoEziLJnv776HJXNAspjlhq5P92iAfR1MzORfRCXZDj+coqpojbKFz18id5pS9Ol1XCwa1nFbaFUzZ11uDakMjquvYhEM2S26zFKIw5cJcorwK9sYksTuItDOHB/2o5bqhc6FNS1Ec5zevLse+2iTOFWQSxZ7GGAn8t0BpCW5z7ufhdt8qHLivcOB2KqelpIgqZBTzxBQSQnApAWAAMK36i4Qqr5L/qNpdS50U7jPNAVddH95+H56wlQp8C+oJu0h4BXBTCIGZYoffYFOuhsrqnYwdQBZ5zoUutveEFXi1alx96DO1x9YOfo9CTvPVYAxM4r36Ma/CNnZkXI3lU9c6aTLD2zlLwkdYvbWt99rlz8XbSi/LxLW/eWqTP+5lRgmVOSMHkwcGYrNp2pQ1lLmBNrIXgUQ66vmTCN2XsstgrOMVm9eWdfITkJVlIjSmwniaVu3S+sRezITSqlNqg5lGUUvuHoBNlygbMawTpkDwju3o/IgoVNThQ/qFEx1Iy5eK7KYdkZksIJAerNLb04WIFLXzVqMjV6fR1tlZoIP4ULJqR57TXqJfE28mUkLYDtoHV7CEOaMPTMZSwrfbbi/aA5J4sKG0msgjkiQCpWmHA5MBmsPGAdZwKiSKqGtvrGM73yWKrqcyb0I7ntAtxSTqshmj54JXX9ZzVACvcPYjdPC5XpM9Q2t+vh3gDF+ispixG8US/akv+AJu2CMC1EIcm2Dlx5Oz5ByKvGOJhWAbQHrT6KMPLp88IttysatvGGBKREKztMxBa2T+2pk0yW3eGGyOsrFC+DpQnUm2DVfriLvPPKgJNM0nj9gIU8dan+9sEgBDGzA00xblt8lzn/b+eOPXpjnd9pmExnI7BEL603c7hq9DaZpn9CbfNAP+d6609rDMw4do7ko3F2NmSHN9/qNtvzGi9Jn3Vq7MNXze3pW5X7PGX7PGn2bWeNbB3iRIaQuhWfARnxb7nmYhh3y/BmY/3wSrhu+JCk7AJvusB8PKg9GVD4Jl5pJVJDttMHmGIUossOy6EPs89PXTk5tKeCkBjBOaQa92+XloWTZ5zULf0NjRr48dSjszReWA1FKKKYm4Jn95vhi2pa/Sh15hbp/nfmbqnLuEiBw52OR3HzZQnVhrxRQSl3KVEkRrH4wgndtmSXnXmbCIxDFKGVUZ71Ie3MJBBQcWuDaPEzSlmt3Sl/3uaiAQzW0/u1FTGdFMoSgn6guZ9K0EF0abYvJnzCIqZYFiQQd70DBQwbg9bEBouWsWA0LOmxa54NrQynkd3WFk+ECnYbRl3Hiue/D52qKUBzfQDHaUMSpRu5jdvgWVz1GCilC2jKLuqXwGC8CABMM9soikqcBUZyEL0imhwAFlIJYUQm9hlTYuzq1CK5PU2ynagJ4OYAsZeUnILH/8cfwKh2z7lbXXSTtzjJwwmvxmk+wzfUQhuLC1cyDs3YaPGguMWO2MAcub7DMB4FzKqtmtcW3nQwk1N+yq2y46Dhykwt3yvO4aRHAgNvQeBX+ZG8cFYfWnA+iN3wFTopDG/swCpB40ljs7aL4lmfn6LFijt3108V8AAAD//wuEi5I=" + return "eJzsW0tv3DgSvvtXFHwMNt57HxbodZKFgU0cOM5ZoKVqmWO2qCGpNjq/fkC9HyTFdlNqDyZ9mrGU+j4Wq4r1oD7CCx43QH4VAq8AFFUMN3C91f9/fQUgkCGRuIEnVOQKIEEZC5oryrMN/OcKAKB8F77ypGBaxI4iS+SmfPQRMrLHTrz+qWOOG0gFL/L6LwaZQzF9UbJ4at+OaNI+bwS/4PGVi/7fjeKrX0W9LxLuPk0gYy4EMhIE8baTZYJSmJFMnY1SiTEBCJS8EDFO5Pc3ZEb6w1TGeLf6kIPFuBY0A9uH7i+tjzVeRSi4qdwGMRf8QBMUC4BqGf/WKDInA2136Oanq8KHQ/42ltaGj0I9c0F/VS4oqjgTCHTblw0D2S18rOiBqiPjqZz1m3HA7GjsSMFUVHrKBnaESfRzty+lb8GOizpeNXxAE7rx9kLMFFVHo+ZMbjOjtzuzPBOFPo2YEbqXEc2ookRhEj0do0JO3MdNzYOe/t2WWNBiwdMRLFg22uA29iFVs/2dQBhMTjCmkdIDZutw+Z8TqjuRxTp0fjiAGjK7grF12HxxIbW6iZ9xT1ZQjRln6HU3H6xexp/+wFgZHlcPojmqvdeiPclzmqX1v7n+cH2O91qXNDgTloge2xkAn5AhY56v4RhWmP4RZlxFYCZbO05DBXW6lE0yliEZ276dQOWzG8e1fX26gjOMiJQ0zfaYqci1peCpzBNWoX8PnCF0FJxWNSKe4K48/saVy1qsO/xxxm6jvD7Jk03gIprs7b+HJnNBs5jmhK1O9nuDfBpNzeRSRCfYDT2eo6hqjrCFzn0jd5pT9up0XS0Y1HJeaVcwZV9vDaoNjaiuYxMO2Sy5zVKIwpQLc4nyJthbk8TuINLOHB70s5brhs6FNi1FcZzfvLkc+26TOFeQSRQHGmMk8M8CpSW4zbmfh9v9qHDgocKBu6mclpIiqpBRzBNTSAjBpQSAAcC06i8SqrxK/pNqdy11UrjPNAdcdX14+318xlYq8B2oZ+wi4Q3AbSEEZood/wXbcjVUVu9k7AiyyHMudLF9IKzAm1Xj6mOfqT22dvAHFHKarwZjYBLv1Y95E7axI+NqLJ+71kmTGd7PWRI+wuqtbb3XLn8u3lZ6WSau/Z+nNvnjXmaUUJkzcjR5YCA226ZNWUOZG2gjexFIpKOeP4vQQym7DMY6XrF5bVknPwFZWSZCYyqMp2nVLq1P7MVMKK06pTaYaRS15O4B2HSJshHDOmEKBO/Yjs6PiEJFHT6kXzjTgbR8qch+2hGZyQIC6cEqvT1diEhRO281OnJ1Gm2dnQU6iI8lq3bkOe0l+jXxZiIlhO2gfXIFS5gz+sBkLCV8u+32oj0giUcbSquJPCJJIlCadjgwGaA5bB1gDadCooi69sY6tvNTouh6KvMmtOcJ3VFMoi6bMXouePVlPUcF8AZnP0EHX+s12TO05ufbAc7wNSqLGbtRLNGf+oav4IY9IUAtxLEJVn48OUsuoch7llgItgGkN40++eDyySOyHRf7+oYBpkQkNEvLHLRG5m+dSZPc5o3B5ihbK4SvA9WZZNtwtY64+8yDmkDTfPKIjTB1rPX5ziYBMLQBQzNtUX7bPPdp7483fm2a022fSWgst0MgpD/9tGP4OpSmeUFv8k0z4G/nSmsPyzx8iOaudHMxZoY01+c/2vYbI0qfee/lylzD56Qrc7/Hf4GRf4//3t/476Kztknc0BZCs+BTNy32I81Czt1+z7D+eUOlGr4nKjgBm+yLHgwrzypXPgiWGRVWkey8WeEF5hqxwLIRQuwjyrcPNG4r4aUEMA5NBu3T5UeUZSXjNZ58R5NAv9ZyKO3M1HkDUksppiTiGsbl+WLYllZHH3qFUXqe+5mpcxQSInLkYJPffWtAdWKtFVNIXMpVShCtfTCCdG6bJeX1Y8IiEscoZVRlvEt5cAsHFRxY4No8TtCUanZL37+7r4FANBfw7EZNZUQzhaIcci9k0ncSXBhtislfMIuolAWKBR3sUcNABeP2sAGh5W4+DAg5Lz/kgmtDK0dodI+R4ZuZhtGOceO57sHne4tSHtxAM9hTxqhE7WJ2+xZUvkQJKkLZMop6oPIFLAADEgwPyCKSpgJTnYUsSKeEAgeUgVhSCL2FVdq4OLcKrUxS76ZoA3o6gC1k5CUhs/zx9+orHLLth89eJ+3MMXLGtPCHTbLPQBCF4MLWzoGw1w0+aywwYrVtfywvl88EgEspq2a3xk2aTyXU3PypbrvoOHCUCvfL87pvEMGB2NB7Evx1bkIWhNV/HUDv/FqWEoU09mcWIPWosdzZQfN5x8wHYcEave2jq78CAAD//196cek=" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index d4326bacd9d..c73c9fd917a 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -5,10 +5,6 @@ description: > Fields for Azure platform logs. fields: - - name: identity - type: group - description: > - Identity - name: operation_name type: keyword description: > From 23f953b30f1210c7d0eb83f185e575687207bcdd Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 16 Nov 2020 12:38:42 +0100 Subject: [PATCH 10/19] add tests --- filebeat/docs/fields.asciidoc | 10 ++++++ x-pack/filebeat/module/azure/fields.go | 2 +- .../azure/platformlogs/_meta/fields.yml | 4 +++ .../azure/platformlogs/ingest/pipeline.yml | 4 +++ .../test/platfformlogs-kube.log-expected.json | 36 +++++++++++++++++++ ...formlogs.log => platformlogs-eventhub.log} | 0 ...> platformlogs-eventhub.log-expected.json} | 2 +- .../platformlogs/test/platformlogs-kube.log | 1 + 8 files changed, 57 insertions(+), 2 deletions(-) create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json rename x-pack/filebeat/module/azure/platformlogs/test/{platformlogs.log => platformlogs-eventhub.log} (100%) rename x-pack/filebeat/module/azure/platformlogs/test/{platformlogs.log-expected.json => platformlogs-eventhub.log-expected.json} (97%) create mode 100644 x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index fd6408bcf5d..96bfb24490e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3108,6 +3108,16 @@ type: keyword Event Category +type: keyword + +-- + +*`azure.platformlogs.status`*:: ++ +-- +Status + + type: keyword -- diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 76a904e0e30..daf5f53e3ed 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW0tv3DgSvvtXFHwMNt57HxbodZKFgU0cOM5ZoKVqmWO2qCGpNjq/fkC9HyTFdlNqDyZ9mrGU+j4Wq4r1oD7CCx43QH4VAq8AFFUMN3C91f9/fQUgkCGRuIEnVOQKIEEZC5oryrMN/OcKAKB8F77ypGBaxI4iS+SmfPQRMrLHTrz+qWOOG0gFL/L6LwaZQzF9UbJ4at+OaNI+bwS/4PGVi/7fjeKrX0W9LxLuPk0gYy4EMhIE8baTZYJSmJFMnY1SiTEBCJS8EDFO5Pc3ZEb6w1TGeLf6kIPFuBY0A9uH7i+tjzVeRSi4qdwGMRf8QBMUC4BqGf/WKDInA2136Oanq8KHQ/42ltaGj0I9c0F/VS4oqjgTCHTblw0D2S18rOiBqiPjqZz1m3HA7GjsSMFUVHrKBnaESfRzty+lb8GOizpeNXxAE7rx9kLMFFVHo+ZMbjOjtzuzPBOFPo2YEbqXEc2ookRhEj0do0JO3MdNzYOe/t2WWNBiwdMRLFg22uA29iFVs/2dQBhMTjCmkdIDZutw+Z8TqjuRxTp0fjiAGjK7grF12HxxIbW6iZ9xT1ZQjRln6HU3H6xexp/+wFgZHlcPojmqvdeiPclzmqX1v7n+cH2O91qXNDgTloge2xkAn5AhY56v4RhWmP4RZlxFYCZbO05DBXW6lE0yliEZ276dQOWzG8e1fX26gjOMiJQ0zfaYqci1peCpzBNWoX8PnCF0FJxWNSKe4K48/saVy1qsO/xxxm6jvD7Jk03gIprs7b+HJnNBs5jmhK1O9nuDfBpNzeRSRCfYDT2eo6hqjrCFzn0jd5pT9up0XS0Y1HJeaVcwZV9vDaoNjaiuYxMO2Sy5zVKIwpQLc4nyJthbk8TuINLOHB70s5brhs6FNi1FcZzfvLkc+26TOFeQSRQHGmMk8M8CpSW4zbmfh9v9qHDgocKBu6mclpIiqpBRzBNTSAjBpQSAAcC06i8SqrxK/pNqdy11UrjPNAdcdX14+318xlYq8B2oZ+wi4Q3AbSEEZood/wXbcjVUVu9k7AiyyHMudLF9IKzAm1Xj6mOfqT22dvAHFHKarwZjYBLv1Y95E7axI+NqLJ+71kmTGd7PWRI+wuqtbb3XLn8u3lZ6WSau/Z+nNvnjXmaUUJkzcjR5YCA226ZNWUOZG2gjexFIpKOeP4vQQym7DMY6XrF5bVknPwFZWSZCYyqMp2nVLq1P7MVMKK06pTaYaRS15O4B2HSJshHDOmEKBO/Yjs6PiEJFHT6kXzjTgbR8qch+2hGZyQIC6cEqvT1diEhRO281OnJ1Gm2dnQU6iI8lq3bkOe0l+jXxZiIlhO2gfXIFS5gz+sBkLCV8u+32oj0giUcbSquJPCJJIlCadjgwGaA5bB1gDadCooi69sY6tvNTouh6KvMmtOcJ3VFMoi6bMXouePVlPUcF8AZnP0EHX+s12TO05ufbAc7wNSqLGbtRLNGf+oav4IY9IUAtxLEJVn48OUsuoch7llgItgGkN40++eDyySOyHRf7+oYBpkQkNEvLHLRG5m+dSZPc5o3B5ihbK4SvA9WZZNtwtY64+8yDmkDTfPKIjTB1rPX5ziYBMLQBQzNtUX7bPPdp7483fm2a022fSWgst0MgpD/9tGP4OpSmeUFv8k0z4G/nSmsPyzx8iOaudHMxZoY01+c/2vYbI0qfee/lylzD56Qrc7/Hf4GRf4//3t/476Kztknc0BZCs+BTNy32I81Czt1+z7D+eUOlGr4nKjgBm+yLHgwrzypXPgiWGRVWkey8WeEF5hqxwLIRQuwjyrcPNG4r4aUEMA5NBu3T5UeUZSXjNZ58R5NAv9ZyKO3M1HkDUksppiTiGsbl+WLYllZHH3qFUXqe+5mpcxQSInLkYJPffWtAdWKtFVNIXMpVShCtfTCCdG6bJeX1Y8IiEscoZVRlvEt5cAsHFRxY4No8TtCUanZL37+7r4FANBfw7EZNZUQzhaIcci9k0ncSXBhtislfMIuolAWKBR3sUcNABeP2sAGh5W4+DAg5Lz/kgmtDK0dodI+R4ZuZhtGOceO57sHne4tSHtxAM9hTxqhE7WJ2+xZUvkQJKkLZMop6oPIFLAADEgwPyCKSpgJTnYUsSKeEAgeUgVhSCL2FVdq4OLcKrUxS76ZoA3o6gC1k5CUhs/zx9+orHLLth89eJ+3MMXLGtPCHTbLPQBCF4MLWzoGw1w0+aywwYrVtfywvl88EgEspq2a3xk2aTyXU3PypbrvoOHCUCvfL87pvEMGB2NB7Evx1bkIWhNV/HUDv/FqWEoU09mcWIPWosdzZQfN5x8wHYcEave2jq78CAAD//196cek=" + return "eJzsW0tv3DgSvvtXFHwMNt57HxbodZKFgU0cOM5ZoKVqmWO2qCGpNjq/fkC9HyTFdlNqDyY6zVid+j4Wq4r1oD7CCx43QH4VAq8AFFUMN3C91f9/fQUgkCGRuIEnVOQKIEEZC5oryrMN/OcKAKD8LXzlScG0iB1FlshN+eojZGSPnXj9qGOOG0gFL/L6LwaZQzF9UbJ4an8d0aR93wh+weMrF/2/G8VXT0W9LxLuPk0gYy4EMhIE8baTZYJSmJFMnY1SiTEBCJS8EDFO5Pc3ZEb6w1TGeLf6kIPFuBY0A9uH7i+tjzVeRSi4qdwGMRf8QBMUC4BqGf/WKDInA2136Oa3q8KHQ/42ltaGj0I9c0F/VS4oqjgTCHTblw0D2S18rOiBqiPjqZz1m3HA7GjsSMFUVHrKBnaESfRzty+lb8GOizpeNXxAE7rx9kLMFFVHo+ZMbjOjtzuzPBOFPo2YEbqXEc2ookRhEj0do0JO3MdNzYOefm5LLGix4OkIFiwbbXAb+5Cq2f5OIAwmJxjTSOkBs3W4/M8J1Z3IYh06PxxADZldwdg6bL64kFrdxM+4Jyuoxowz9LqbD1Yv409/YKwMr6sX0RzV3s+iPclzmqX1v7n+cH2O91qXNDgTloge2xkAn5AhY56v4RhWmP4RZlxFYCZbO05DBXW6lE0yliEZ276dQOWzG8e1fX26gjOMiJQ0zfaYqci1peCpzBNWoZ8HzhA6Ck6rGhFPcFcef+PKZS3WHf44Y7dRXp/kySZwEU329t9Dk7mgWUxzwlYn+71BPo2mZnIpohPshh7PUVQ1R9hC576RO80pe3W6rhYMajmvtCuYsq+3BtWGRlTXsQmHbJbcZilEYcqFuUR5E+ytSWJ3EGlnDg/6Wct1Q+dCm5aiOM5v3lyOfbdJnCvIJIoDjTES+GeB0hLc5tzPw+1+VDjwUOHA3VROS0kRVcgo5okpJITgUgLAAGBa9RcJVV4l/0m1u5Y6KdxnmgOuuj68/T4+YysV+A7UM3aR8AbgthACM8WO/4JtuRoqq99k7AiyyHMudLF9IKzAm1Xj6mOfqT22dvAHFHKarwZjYBLv1Y95E7axI+NqLJ+71kmTGd7PWRI+wuqtbb3XLn8u3lZ6WSau/Z+nNvnjXmaUUJkzcjR5YCA226ZNWUOZG2gjexFIpKOeP4vQQym7DMY6XrF5bVknPwFZWSZCYyqMp2nVLq1P7MVMKK06pTaYaRS15O4B2HSJshHDOmEKBO/Yjs6PiEJFHT6kf3CmA2n5UpH9tCMykwUE0oNVenu6EJGidt5qdOTqNNo6Owt0EB9LVu3Ic9pL9GvizURKCNtB++QKljBn9IHJWEr4dtvtRXtAEo82lFYTeUSSRKA07XBgMkBz2DrAGk6FRBF17Y11bOenRNH1VOZNaM8TuqOYRF02Y/Rc8OrLeo4K4A3OfoIOvtZrsmdozePbAc7wNSqLGbtRLNGf+oav4IY9IUAtxLEJVn48OUsuoch7llgItgGkN40++eDyySOyHRf7+oYBpkQkNEvLHLRG5m+dSZPc5o3B5ihbK4SvA9WZZNtwtY64+8yDmkDTfPKIjTB1rPX5ziYBMLQBQzNtUX7bPPdp7483fm2a022fSWgst0MgpD/9tGP4OpSmeUFv8k0z4G/nSmsPyzx8iOaudHMxZoY01+c/2vYbI0qfee/lylzD56Qrc7/Hf4GRf4//+gOucJA/pvLe46TxomO9SYjSxkiz4AM+LfYjzUKO+H6Py/5586savicqOAGb7IueQSuPRVc+c5aZSlaR7Lyx5AVGKLHAsudC7NPQt89ObivhpQQwzmcGndrlp6Fl0eQ1CX1HQ0e/LnYo7cyUlANSSymmJOKa++X5YtiWrkofeoWpfZ77malz6hIicuRgk9991kB1Dq8VU0hcylVKEK19MIJ0bpsl5U1nwiISxyhlZMzzQ9Hq4KCCM5cVvTxO0JRqdktf9buvgUA0d/3sRk1lRDOFopynL2TSdxJcGG2KyV8wi6iUBYoFHexRw0AF4/awAaHlLlkMCDnvWeSCa0Mrp3V0j5Hh85yG0Y5x47nuwed7i1Ie3EAz2FPGqETtYnb7FlS+RAkqQtkyinqg8gUsAAMSDA/IIpKmAlOdhSxIp4QCB5SBWFIIvYVV2rg4twqtTFLvpmgDejqALWTkJSGz/PGn8Sscsu031l4n7cwxcsZg0tA4qh6f2SMKwYWtnQNhbzZ81lhgxGonDFjeY58JAJdSVs1ujUs7n0qouVFX3XbRceAoFe6X53XfIIIDsaH3JPjr3DAuCKv/OoDe+Q0wJQpp7M8sQOpRY7mzg+ZLkplvz4I1ettXV38FAAD//4amit4=" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index c73c9fd917a..66ccb4a53a7 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -25,6 +25,10 @@ type: keyword description: > Event Category + - name: status + type: keyword + description: > + Status - name: properties type: group description: > diff --git a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml index 5bc67bdc336..8493ef886fe 100644 --- a/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml +++ b/x-pack/filebeat/module/azure/platformlogs/ingest/pipeline.yml @@ -134,6 +134,10 @@ processors: field: azure.platformlogs.properties.statusCode target_field: azure.platformlogs.properties.status_code ignore_missing: true +- rename: + field: azure.platformlogs.Status + target_field: azure.platformlogs.status + ignore_missing: true - geoip: field: source.ip target_field: geo diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json new file mode 100644 index 00000000000..8463ab7c5a5 --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json @@ -0,0 +1,36 @@ +{ + "cloud":{ + "provider":"azure" + }, + "@timestamp":"2020-11-09T10:57:31.000Z", + "event":{ + "action":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "ingested":"2020-11-16T11:31:07.522993300Z", + "kind":"event" + }, + "message" : """{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe","stage":"ResponseComplete","requestURI":"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s","verb":"get","user":{"username":"system:serviceaccount:kube-system:resourcequota-controller","uid":"33917550-525b-11ea-a4aa-168268435a05","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["172.31.38.231"],"userAgent":"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller","responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2020-11-09T10:57:31.643810Z","stageTimestamp":"2020-11-09T10:57:31.643954Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}}""", + "azure":{ + "subscription_id":"70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "resource":{ + "name":"OBSKUBE", + "id":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "provider":"MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "group":"OBS-INFRASTRUCTURE" + }, + "platformlogs":{ + "ccpNamespace":"5e4bf4baee195b00017cdbfa", + "operation_name":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "Cloud":"AzureCloud", + "Environment":"prod", + "UnderlayClass":"hcp-underlay", + "UnderlayName":"hcp-underlay-westeurope-cx-316", + "category":"kube-audit", + "event_category":"Administrative", + "properties":{ + "pod":"kube-apiserver-666bd4b459-hjgdc", + "stream":"stdout" + } + } + } +} + diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log similarity index 100% rename from x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log rename to x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json similarity index 97% rename from x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json rename to x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json index 5455bd306a2..07c9bb799cc 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -20,7 +20,7 @@ "group" : "OBS-TEST" }, "platformlogs" : { - "Status" : "Succeeded", + "status" : "Succeeded", "SubscriptionId" : "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", "Caller" : "Portal", "ActivityId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8", diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log new file mode 100644 index 00000000000..ee36162dc2d --- /dev/null +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log @@ -0,0 +1 @@ +{"Cloud":"AzureCloud","Environment":"prod","UnderlayClass":"hcp-underlay","UnderlayName":"hcp-underlay-westeurope-cx-316","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:kube-system:resourcequota-controller\",\"uid\":\"33917550-525b-11ea-a4aa-168268435a05\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"]},\"sourceIPs\":[\"172.31.38.231\"],\"userAgent\":\"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller\",\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2020-11-09T10:57:31.643810Z\",\"stageTimestamp\":\"2020-11-09T10:57:31.643954Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}}\n","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} From 67953dc865e154933e05f937693bb13e3bab5f1d Mon Sep 17 00:00:00 2001 From: Mariana Date: Mon, 16 Nov 2020 17:04:34 +0100 Subject: [PATCH 11/19] add mapping --- filebeat/docs/fields.asciidoc | 60 +++++++++++++++++++ x-pack/filebeat/module/azure/fields.go | 2 +- .../azure/platformlogs/_meta/fields.yml | 25 +++++++- .../test/platfformlogs-kube.log-expected.json | 2 - .../platformlogs-eventhub.log-expected.json | 1 - 5 files changed, 85 insertions(+), 5 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 96bfb24490e..73993012444 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3118,6 +3118,66 @@ type: keyword Status +type: keyword + +-- + +*`azure.platformlogs.ccpNamespace`*:: ++ +-- +ccpNamespace + + +type: keyword + +-- + +*`azure.platformlogs.Cloud`*:: ++ +-- +Cloud + + +type: keyword + +-- + +*`azure.platformlogs.Environment`*:: ++ +-- +Environment + + +type: keyword + +-- + +*`azure.platformlogs.Caller`*:: ++ +-- +Caller + + +type: keyword + +-- + +*`azure.platformlogs.ScaleUnit`*:: ++ +-- +ScaleUnit + + +type: keyword + +-- + +*`azure.platformlogs.ActivityId`*:: ++ +-- +ActivityId + + type: keyword -- diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index daf5f53e3ed..8db4bb20436 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsW0tv3DgSvvtXFHwMNt57HxbodZKFgU0cOM5ZoKVqmWO2qCGpNjq/fkC9HyTFdlNqDyY6zVid+j4Wq4r1oD7CCx43QH4VAq8AFFUMN3C91f9/fQUgkCGRuIEnVOQKIEEZC5oryrMN/OcKAKD8LXzlScG0iB1FlshN+eojZGSPnXj9qGOOG0gFL/L6LwaZQzF9UbJ4an8d0aR93wh+weMrF/2/G8VXT0W9LxLuPk0gYy4EMhIE8baTZYJSmJFMnY1SiTEBCJS8EDFO5Pc3ZEb6w1TGeLf6kIPFuBY0A9uH7i+tjzVeRSi4qdwGMRf8QBMUC4BqGf/WKDInA2136Oa3q8KHQ/42ltaGj0I9c0F/VS4oqjgTCHTblw0D2S18rOiBqiPjqZz1m3HA7GjsSMFUVHrKBnaESfRzty+lb8GOizpeNXxAE7rx9kLMFFVHo+ZMbjOjtzuzPBOFPo2YEbqXEc2ookRhEj0do0JO3MdNzYOefm5LLGix4OkIFiwbbXAb+5Cq2f5OIAwmJxjTSOkBs3W4/M8J1Z3IYh06PxxADZldwdg6bL64kFrdxM+4Jyuoxowz9LqbD1Yv409/YKwMr6sX0RzV3s+iPclzmqX1v7n+cH2O91qXNDgTloge2xkAn5AhY56v4RhWmP4RZlxFYCZbO05DBXW6lE0yliEZ276dQOWzG8e1fX26gjOMiJQ0zfaYqci1peCpzBNWoZ8HzhA6Ck6rGhFPcFcef+PKZS3WHf44Y7dRXp/kySZwEU329t9Dk7mgWUxzwlYn+71BPo2mZnIpohPshh7PUVQ1R9hC576RO80pe3W6rhYMajmvtCuYsq+3BtWGRlTXsQmHbJbcZilEYcqFuUR5E+ytSWJ3EGlnDg/6Wct1Q+dCm5aiOM5v3lyOfbdJnCvIJIoDjTES+GeB0hLc5tzPw+1+VDjwUOHA3VROS0kRVcgo5okpJITgUgLAAGBa9RcJVV4l/0m1u5Y6KdxnmgOuuj68/T4+YysV+A7UM3aR8AbgthACM8WO/4JtuRoqq99k7AiyyHMudLF9IKzAm1Xj6mOfqT22dvAHFHKarwZjYBLv1Y95E7axI+NqLJ+71kmTGd7PWRI+wuqtbb3XLn8u3lZ6WSau/Z+nNvnjXmaUUJkzcjR5YCA226ZNWUOZG2gjexFIpKOeP4vQQym7DMY6XrF5bVknPwFZWSZCYyqMp2nVLq1P7MVMKK06pTaYaRS15O4B2HSJshHDOmEKBO/Yjs6PiEJFHT6kf3CmA2n5UpH9tCMykwUE0oNVenu6EJGidt5qdOTqNNo6Owt0EB9LVu3Ic9pL9GvizURKCNtB++QKljBn9IHJWEr4dtvtRXtAEo82lFYTeUSSRKA07XBgMkBz2DrAGk6FRBF17Y11bOenRNH1VOZNaM8TuqOYRF02Y/Rc8OrLeo4K4A3OfoIOvtZrsmdozePbAc7wNSqLGbtRLNGf+oav4IY9IUAtxLEJVn48OUsuoch7llgItgGkN40++eDyySOyHRf7+oYBpkQkNEvLHLRG5m+dSZPc5o3B5ihbK4SvA9WZZNtwtY64+8yDmkDTfPKIjTB1rPX5ziYBMLQBQzNtUX7bPPdp7483fm2a022fSWgst0MgpD/9tGP4OpSmeUFv8k0z4G/nSmsPyzx8iOaudHMxZoY01+c/2vYbI0qfee/lylzD56Qrc7/Hf4GRf4//+gOucJA/pvLe46TxomO9SYjSxkiz4AM+LfYjzUKO+H6Py/5586savicqOAGb7IueQSuPRVc+c5aZSlaR7Lyx5AVGKLHAsudC7NPQt89ObivhpQQwzmcGndrlp6Fl0eQ1CX1HQ0e/LnYo7cyUlANSSymmJOKa++X5YtiWrkofeoWpfZ77malz6hIicuRgk9991kB1Dq8VU0hcylVKEK19MIJ0bpsl5U1nwiISxyhlZMzzQ9Hq4KCCM5cVvTxO0JRqdktf9buvgUA0d/3sRk1lRDOFopynL2TSdxJcGG2KyV8wi6iUBYoFHexRw0AF4/awAaHlLlkMCDnvWeSCa0Mrp3V0j5Hh85yG0Y5x47nuwed7i1Ie3EAz2FPGqETtYnb7FlS+RAkqQtkyinqg8gUsAAMSDA/IIpKmAlOdhSxIp4QCB5SBWFIIvYVV2rg4twqtTFLvpmgDejqALWTkJSGz/PGn8Sscsu031l4n7cwxcsZg0tA4qh6f2SMKwYWtnQNhbzZ81lhgxGonDFjeY58JAJdSVs1ujUs7n0qouVFX3XbRceAoFe6X53XfIIIDsaH3JPjr3DAuCKv/OoDe+Q0wJQpp7M8sQOpRY7mzg+ZLkplvz4I1ettXV38FAAD//4amit4=" + return "eJzsXEtv4zgSvudXFHJsbGfvOSzgTacXAbY7jSR9FhiprHBCixqScuD+9QNS1ssiKTqm5AymdZqJPN/38VHFelDzGV5xdw3kVyXwAkBRxfAaLlf63y8vAAQyJBKv4RkVuQDIUKaClory4hr+cwEAYH4L33hWMQ2xpsgyeW1efYaCbLCD14/alXgNueBVuf+LBXMI04eS1XP764Rm7fsG+BV3b1z0/26Fr59aeh8S7r6MKFMuBDIShfGmw7JRKSxIoU5mqWFsBAIlr0SKI/z+gkygP4wxDlerTzkYjG9AE7R96v7Q+lyHo4hFN8ZtGEvBtzRDMQOpxvi3ZpElGcx2x25/uyh9PObvh2it+6jUCxf0V22CovYzkUhXfWwYYLf0qaJbqnaM53LSbg4dZidjTSqmEmMp17AmTGKYuX01tgVrLvb+qtEDWtBVsBVioajaWWfOZjYT83Znx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyHz80gLk6efGcEHLBc87cHC5ZIN/sw+l2vffEYLBZgSHMnK6xWIZLf/zUnUnslhGzqOHqBGzrhhbRs1XH1M7N+kLbsgCU2PnGVrd1SenlfHnPzBVltf1i2RKau9nyYaUJS3y/X9z+enyFOt1DmlwJszhPVYTBCEuQ6a8XMIwnDT9I8w6ishKVm6eRgrqcKkYRSxDMa51O0LKrZ/Ht3x9uYIzTIiUNC82WKjEt6QQOJlHjEI/D5whdBK8u+pAeIZrc/wdZi5Lqe74DyN2l+TlRR69Bc4yk731D5jJUtAipSVhi4v90TAfJ1MrOZfQEXcjj5co6pwjbqJz3+COY8penq6zBcu0nJbaVUy5x7sn1RuNqK5iE4/ZjtxGKURhzoU9RXkX7Y0NsTuItDHHJ73VuH7qUuitpSgexjfvTsd+uBCnEjKJYktTTAT+WaF0OLcp8wswu8eaBx5qHrgb47SSFFGVTFKe2VxCDC2GAAYE46y/yqgKSvmPyt016ihxnygO+PL6+Pv36QVbVOBrUC/YecIrgJtKCCwU2/0LVmY0VNa/KdgOZFWWXOhke0tYhVeL+tWnvlK3b+3otyjkOF6NpsAGH1SPeRe3tSLjKyyfOtZRkRk+zlkS38PqpW2t140/5W/reZnHr/2f5y78w1pmklFZMrKzWWAkNaumTLmnshfQDvaLQCI9+fxJgh4MtnHG2l+x6dlydn4iqnJ0hA6lMJ7ndbl0f2LPtoXyulLqohl7UUfsHkFNFyhbOZwdpkj0nuXo7IgoVNRjQ/oHJxqQxpeKbMYVkYkoINI8ONHb04WIHLXx1q0jX6XRVdmZoYL4ZFS1Lc9xLTGsiDfhKSFuBe2Lz1nC1KaPLMaRwrfL7k7aI4p4crG0M1EmJMsEStsKRxYDtISVh6zRVEkUSVfeWGbv/JQouprK9Bba8IyuKWZJF81YLReC6rKBrQJ4h7EfMQff9mNyR2jNE1oBLvAtMcmMe1PMUZ/6jm/gpz3CQc2ksXFWYTo5y84xkfcscwhsHUivG330wRUSRxRrLjb7GwaYE5HRIjcx6J6Zv7cnTUqXNUbro6ycFKEGtI8k24Krs8XdVx51CzTFpwDfCGPDWl7vZBAAwz1gKabNqm9VliHl/cOFX1rmeNknAhrH7RCIaU8/3RyhBqVlntGaQsMM+NuZ0tLNsgAboqUv3JxNmSXMDfmHtvzGiNJn3ke5MtfoOerK3O/2X2Tm3+2/foMrHuXjGK+d1LT8Hv+OsBO1ob1hvIrYahjDNUS3xZYKbu5FRFxCB2g7OsJYzBvfFryG6jElDH8WNOLo7JANYVN0HMVqp9yxtmN+xG74h2g9az9Ji+i9Zw37mRYxu8+/O7n/vNbqnr4HFV2AC/us4dHCHfuFw6F5Gua1JzutY36G7l4q0JQDibtR//623k0NbhDA2jocNBHmb9SbfD6oSf+B+uFhDZZYszNR7RiImmtijBBfS7osZ+N2FPz61AtcKCnLsG3qbQjG8BwluPC7L26oTi/1xFQS5zIVQ6JnH6wkndkWmbmET1hC0hSlTKwpaCxZHR3UdPaMtxfHCZpTrW7uW6j3eyIQzTVU96amMqGFQmGuesy0pe8k+DjaEJO/YpFQKSsUMxrYk6aBmsZvYQNB893/GQjyXgEqBdcbzTSS6QYTy5djjaI149ZzPUDPj5bFHNxAC9hQxqhEbWLu/S2ofE0yVISyeSbqgcpXcBAMRDDcIktIngvMdRQyoxxDBR4qi7CsEnoJ67Bxdm01mwlS78ZsA3nagc20yY0gO/7h/7VhgUO2/fw/6KSdOEZO6Jlbapr1E9IWRyG4cFVxIO6lm1vNBVautvmF5hOLCQdwrsnaq1viPtkXQzXVhd2XXbQf2EmFm/l13TeM4GFs5D0L/jbVJ46i6r8eog9+OVGJSlrrMzOIetJc/uig+chp4rPIaPXd9tXFXwEAAP//4v0q+Q==" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index 66ccb4a53a7..3b583fa88b0 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -29,6 +29,30 @@ type: keyword description: > Status + - name: ccpNamespace + type: keyword + description: > + ccpNamespace + - name: Cloud + type: keyword + description: > + Cloud + - name: Environment + type: keyword + description: > + Environment + - name: Caller + type: keyword + description: > + Caller + - name: ScaleUnit + type: keyword + description: > + ScaleUnit + - name: ActivityId + type: keyword + description: > + ActivityId - name: properties type: group description: > @@ -39,4 +63,3 @@ description: > Status code - diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json index 8463ab7c5a5..c8aa29efa67 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json @@ -22,8 +22,6 @@ "operation_name":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", "Cloud":"AzureCloud", "Environment":"prod", - "UnderlayClass":"hcp-underlay", - "UnderlayName":"hcp-underlay-westeurope-cx-316", "category":"kube-audit", "event_category":"Administrative", "properties":{ diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json index 07c9bb799cc..10aa61d6db5 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -21,7 +21,6 @@ }, "platformlogs" : { "status" : "Succeeded", - "SubscriptionId" : "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", "Caller" : "Portal", "ActivityId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8", "EventTimeString" : "11/3/2020 9:06:42 AM +00:00", From b1b304254970a46f8e30411c5fa5460b52ed771e Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 17 Nov 2020 13:27:37 +0100 Subject: [PATCH 12/19] test --- .../module/azure/platformlogs/test/platformlogs-eventhub.log | 2 +- .../module/azure/platformlogs/test/platformlogs-kube.log | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log index 80af31aa454..13f18cfe2c2 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log @@ -1 +1 @@ -{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","SubscriptionId":"7657426d-c4c3-44ac-88a2-3b2cd59e6dba","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"} +{"ActivityId":"30ed877c-a36b-491a-bd4d-ddd847fe55b8","Caller":"Portal","Environment":"PROD","EventName":"Retreive ConsumerGroup","EventProperties":"{\"SubscriptionId\":\"7657426d-c4c3-44ac-88a2-3b2cd59e6dba\",\"Namespace\":\"obstesteventhubs\",\"Via\":\"sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04\u0026$skip=0\u0026$top=100\",\"TrackingId\":\"30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2\"}","EventTimeString":"11/3/2020 9:06:42 AM +00:00","Region":"West Europe","ScaleUnit":"PROD-AM3-AZ501","Status":"Succeeded","category":"OperationalLogs","resourceId":"/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log index ee36162dc2d..a232f6254fb 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log @@ -1 +1 @@ -{"Cloud":"AzureCloud","Environment":"prod","UnderlayClass":"hcp-underlay","UnderlayName":"hcp-underlay-westeurope-cx-316","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:kube-system:resourcequota-controller\",\"uid\":\"33917550-525b-11ea-a4aa-168268435a05\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"]},\"sourceIPs\":[\"172.31.38.231\"],\"userAgent\":\"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller\",\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2020-11-09T10:57:31.643810Z\",\"stageTimestamp\":\"2020-11-09T10:57:31.643954Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}}\n","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} +{"Cloud":"AzureCloud","Environment":"prod","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:kube-system:resourcequota-controller\",\"uid\":\"33917550-525b-11ea-a4aa-168268435a05\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"]},\"sourceIPs\":[\"172.31.38.231\"],\"userAgent\":\"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller\",\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2020-11-09T10:57:31.643810Z\",\"stageTimestamp\":\"2020-11-09T10:57:31.643954Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}}\n","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} From 227f6ccd3260667c99e1ab0959e58781f05d271e Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 17 Nov 2020 15:55:11 +0100 Subject: [PATCH 13/19] update mapping --- filebeat/docs/fields.asciidoc | 13 +++---------- x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/platformlogs/_meta/fields.yml | 11 ++++------- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 73993012444..cec2daac3c6 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3182,20 +3182,13 @@ type: keyword -- -[float] -=== properties - -Properties - - - -*`azure.platformlogs.properties.status_code`*:: +*`azure.platformlogs.properties.*`*:: + -- -Status code +Properties -type: keyword +type: object -- diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 8db4bb20436..9cdb45d510e 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsXEtv4zgSvudXFHJsbGfvOSzgTacXAbY7jSR9FhiprHBCixqScuD+9QNS1ssiKTqm5AymdZqJPN/38VHFelDzGV5xdw3kVyXwAkBRxfAaLlf63y8vAAQyJBKv4RkVuQDIUKaClory4hr+cwEAYH4L33hWMQ2xpsgyeW1efYaCbLCD14/alXgNueBVuf+LBXMI04eS1XP764Rm7fsG+BV3b1z0/26Fr59aeh8S7r6MKFMuBDIShfGmw7JRKSxIoU5mqWFsBAIlr0SKI/z+gkygP4wxDlerTzkYjG9AE7R96v7Q+lyHo4hFN8ZtGEvBtzRDMQOpxvi3ZpElGcx2x25/uyh9PObvh2it+6jUCxf0V22CovYzkUhXfWwYYLf0qaJbqnaM53LSbg4dZidjTSqmEmMp17AmTGKYuX01tgVrLvb+qtEDWtBVsBVioajaWWfOZjYT83Znx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyHz80gLk6efGcEHLBc87cHC5ZIN/sw+l2vffEYLBZgSHMnK6xWIZLf/zUnUnslhGzqOHqBGzrhhbRs1XH1M7N+kLbsgCU2PnGVrd1SenlfHnPzBVltf1i2RKau9nyYaUJS3y/X9z+enyFOt1DmlwJszhPVYTBCEuQ6a8XMIwnDT9I8w6ishKVm6eRgrqcKkYRSxDMa51O0LKrZ/Ht3x9uYIzTIiUNC82WKjEt6QQOJlHjEI/D5whdBK8u+pAeIZrc/wdZi5Lqe74DyN2l+TlRR69Bc4yk731D5jJUtAipSVhi4v90TAfJ1MrOZfQEXcjj5co6pwjbqJz3+COY8penq6zBcu0nJbaVUy5x7sn1RuNqK5iE4/ZjtxGKURhzoU9RXkX7Y0NsTuItDHHJ73VuH7qUuitpSgexjfvTsd+uBCnEjKJYktTTAT+WaF0OLcp8wswu8eaBx5qHrgb47SSFFGVTFKe2VxCDC2GAAYE46y/yqgKSvmPyt016ihxnygO+PL6+Pv36QVbVOBrUC/YecIrgJtKCCwU2/0LVmY0VNa/KdgOZFWWXOhke0tYhVeL+tWnvlK3b+3otyjkOF6NpsAGH1SPeRe3tSLjKyyfOtZRkRk+zlkS38PqpW2t140/5W/reZnHr/2f5y78w1pmklFZMrKzWWAkNaumTLmnshfQDvaLQCI9+fxJgh4MtnHG2l+x6dlydn4iqnJ0hA6lMJ7ndbl0f2LPtoXyulLqohl7UUfsHkFNFyhbOZwdpkj0nuXo7IgoVNRjQ/oHJxqQxpeKbMYVkYkoINI8ONHb04WIHLXx1q0jX6XRVdmZoYL4ZFS1Lc9xLTGsiDfhKSFuBe2Lz1nC1KaPLMaRwrfL7k7aI4p4crG0M1EmJMsEStsKRxYDtISVh6zRVEkUSVfeWGbv/JQouprK9Bba8IyuKWZJF81YLReC6rKBrQJ4h7EfMQff9mNyR2jNE1oBLvAtMcmMe1PMUZ/6jm/gpz3CQc2ksXFWYTo5y84xkfcscwhsHUivG330wRUSRxRrLjb7GwaYE5HRIjcx6J6Zv7cnTUqXNUbro6ycFKEGtI8k24Krs8XdVx51CzTFpwDfCGPDWl7vZBAAwz1gKabNqm9VliHl/cOFX1rmeNknAhrH7RCIaU8/3RyhBqVlntGaQsMM+NuZ0tLNsgAboqUv3JxNmSXMDfmHtvzGiNJn3ke5MtfoOerK3O/2X2Tm3+2/foMrHuXjGK+d1LT8Hv+OsBO1ob1hvIrYahjDNUS3xZYKbu5FRFxCB2g7OsJYzBvfFryG6jElDH8WNOLo7JANYVN0HMVqp9yxtmN+xG74h2g9az9Ji+i9Zw37mRYxu8+/O7n/vNbqnr4HFV2AC/us4dHCHfuFw6F5Gua1JzutY36G7l4q0JQDibtR//623k0NbhDA2jocNBHmb9SbfD6oSf+B+uFhDZZYszNR7RiImmtijBBfS7osZ+N2FPz61AtcKCnLsG3qbQjG8BwluPC7L26oTi/1xFQS5zIVQ6JnH6wkndkWmbmET1hC0hSlTKwpaCxZHR3UdPaMtxfHCZpTrW7uW6j3eyIQzTVU96amMqGFQmGuesy0pe8k+DjaEJO/YpFQKSsUMxrYk6aBmsZvYQNB893/GQjyXgEqBdcbzTSS6QYTy5djjaI149ZzPUDPj5bFHNxAC9hQxqhEbWLu/S2ofE0yVISyeSbqgcpXcBAMRDDcIktIngvMdRQyoxxDBR4qi7CsEnoJ67Bxdm01mwlS78ZsA3nagc20yY0gO/7h/7VhgUO2/fw/6KSdOEZO6Jlbapr1E9IWRyG4cFVxIO6lm1vNBVautvmF5hOLCQdwrsnaq1viPtkXQzXVhd2XXbQf2EmFm/l13TeM4GFs5D0L/jbVJ46i6r8eog9+OVGJSlrrMzOIetJc/uig+chp4rPIaPXd9tXFXwEAAP//4v0q+Q==" + return "eJzsXEtv4zgSvudXFHJsbGfvOSzgTacXAbY7jSR9FhiprHBCixqScuD+9QNS1ssiKTqm5AymdZqJPN/38VHFelDzGV5xdw3kVyXwAkBRxfAaLlf63y8vAAQyJBKv4RkVuQDIUKaClory4hr+cwEAYH4L33hWMQ2xpsgyeW1efYaCbLCD14/alXgNueBVuf+LBXMI04eS1XP764Rm7fsG+BV3b1z0/26Fr59aeh8S7r6MKFMuBDIShfGmw7JRKSxIoU5mqWFsBAIlr0SKI/z+gkygP4wxDlerTzkYjG9AE7R96v7Q+lyHo4hFN8ZtGEvBtzRDMQOpxvi3ZpElGcx2x25/uyh9PObvh2it+6jUCxf0V22CovYzkUhXfWwYYLf0qaJbqnaM53LSbg4dZidjTSqmEmMp17AmTGKYuX01tgVrLvb+qtEDWtBVsBVioajaWWfOZjYT83Znx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyHz80gLk6efGcEHLBc87cHC5ZIN/sw+l2vffEYLBZgSHMnK6xWIZLf/zUnUnslhGzqOHqBGzrhhbRs1XH1M7N+kLbsgCU2PnGVrd1SenlfHnPzBVltf1i2RKau9nyYaUJS3y/X9z+enyFOt1DmlwJszhPVYTBCEuQ6a8XMIwnDT9I8w6ishKVm6eRgrqcKkYRSxDMa51O0LKrZ/Ht3x9uYIzTIiUNC82WKjEt6QQOJlHjEI/D5whdBK8u+pAeIZrc/wdZi5Lqe74DyN2l+TlRR69Bc4yk731D5jJUtAipSVhi4v90TAfJ1MrOZfQEXcjj5co6pwjbqJz3+COY8penq6zBcu0nJbaVUy5x7sn1RuNqK5iE4/ZjtxGKURhzoU9RXkX7Y0NsTuItDHHJ73VuH7qUuitpSgexjfvTsd+uBCnEjKJYktTTAT+WaF0OLcp8wswu8eaBx5qHrgb47SSFFGVTFKe2VxCDC2GAAYE46y/yqgKSvmPyt016ihxnygO+PL6+Pv36QVbVOBrUC/YecIrgJtKCCwU2/0LVmY0VNa/KdgOZFWWXOhke0tYhVeL+tWnvlK3b+3otyjkOF6NpsAGH1SPeRe3tSLjKyyfOtZRkRk+zlkS38PqpW2t140/5W/reZnHr/2f5y78w1pmklFZMrKzWWAkNaumTLmnshfQDvaLQCI9+fxJgh4MtnHG2l+x6dlydn4iqnJ0hA6lMJ7ndbl0f2LPtoXyulLqohl7UUfsHkFNFyhbOZwdpkj0nuXo7IgoVNRjQ/oHJxqQxpeKbMYVkYkoINI8ONHb04WIHLXx1q0jX6XRVdmZoYL4ZFS1Lc9xLTGsiDfhKSFuBe2Lz1nC1KaPLMaRwrfL7k7aI4p4crG0M1EmJMsEStsKRxYDtISVh6zRVEkUSVfeWGbv/JQouprK9Bba8IyuKWZJF81YLReC6rKBrQJ4h7EfMQff9mNyR2jNE1oBLvAtMcmMe1PMUZ/6jm/gpz3CQc2ksXFWYTo5y84xkfcscwhsHUivG330wRUSRxRrLjb7GwaYE5HRIjcx6J6Zv7cnTUqXNUbro6ycFKEGtI8k24Krs8XdVx51CzTFpwDfCGPDWl7vZBAAwz1gKabNqm9VliHl/cOFX1rmeNknAhrH7RCIaU8/3RyhBqVlntGaQsMM+NuZ0tLNsgAboqUv3JxNmSXMDfmHtvzGiNJn3ke5MtfoOerK3O/2X2Tm3+2/foMrHuXjGK+d1LT8Hv+OsBO1ob1hvIrYahjDNUS3xZYKbu5FRFxCB2g7OsJYzBvfFryG6jElDH8WNOLo7JANYVN0HMVqp9yxtmOOezWj6oPndmJAueGYMsPx/fXRoafdGy2it4w17GdaxGwa/27A/vM6onv6HlR0AS7ss0Y1CzfaF45i5ulz157stEb3GZpyqUBTxSPu/vr7u3E3NbhBAGvHb1D7n7+/btLwoN76B2pjh/VFYs3ORJFiIGquiTFCfJ3kspyN21Gn61MvcA+kLMO2qbePF8NzlODC7z6UoTor1BNTSZzLVAyJnn2wknRmW2Tm7jxhCUlTlDKxZo6xZHV0UNPZE9VeHCdoTrW6uS+P3u+JQDS3R92bmsqEFgqFuaEx05a+k+DjaENM/opFQqWsUMxoYE+aBmoav4UNBM13bWcgyHtzpxRcbzSTmNENJpYPvhpFa8at53qAnh8tizm4gRawoYxRidrE3PtbUPmaZKgIZfNM1AOVr+AgGIhguEWWkDwXmOsoZEY5hgo8VBZhWSX0EtZh4+zaajYTpN6N2QbytAObaZMbQXb8w//ZwgKHbPvVftBJO3GMnNDqtpQi6yekm41CcOG69w9x78rcai6wcrU9KzRfRkw4gHNN1l7dEtfAvhiqqebpvuyi/cBOKtzMr+u+YQQPYyPvWfC3qfZuFFX/9RB98DuFSlTSWp+ZQdST5vJHB823SRNfM0b7Iqh9dfFXAAAA//+vNh7N" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index 3b583fa88b0..14f27984dde 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -53,13 +53,10 @@ type: keyword description: > ActivityId - - name: properties - type: group + - name: properties.* + type: object + object_type: keyword + object_type_mapping_type: "*" description: > Properties - fields: - - name: status_code - type: keyword - description: > - Status code From bfbf01419b7d485a3c4a2da39d026b1c3689e2d2 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 17 Nov 2020 17:09:49 +0100 Subject: [PATCH 14/19] fix file name --- ...kube.log-expected.json => platformlogs-kube.log-expected.json} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename x-pack/filebeat/module/azure/platformlogs/test/{platfformlogs-kube.log-expected.json => platformlogs-kube.log-expected.json} (100%) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json similarity index 100% rename from x-pack/filebeat/module/azure/platformlogs/test/platfformlogs-kube.log-expected.json rename to x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json From 65f00162f75c6084eced63e8157b2fc8ae0f0a6a Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 17 Nov 2020 19:49:30 +0100 Subject: [PATCH 15/19] update file --- .../module/azure/platformlogs/test/platformlogs-kube.log | 2 +- .../azure/platformlogs/test/platformlogs-kube.log-expected.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log index a232f6254fb..7b8930fb341 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log @@ -1 +1 @@ -{"Cloud":"AzureCloud","Environment":"prod","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s\",\"verb\":\"get\",\"user\":{\"username\":\"system:serviceaccount:kube-system:resourcequota-controller\",\"uid\":\"33917550-525b-11ea-a4aa-168268435a05\",\"groups\":[\"system:serviceaccounts\",\"system:serviceaccounts:kube-system\",\"system:authenticated\"]},\"sourceIPs\":[\"172.31.38.231\"],\"userAgent\":\"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller\",\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2020-11-09T10:57:31.643810Z\",\"stageTimestamp\":\"2020-11-09T10:57:31.643954Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"RBAC: allowed by ClusterRoleBinding \\\"system:discovery\\\" of ClusterRole \\\"system:discovery\\\" to Group \\\"system:authenticated\\\"\"}}\n","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} +{"Cloud":"AzureCloud","Environment":"prod","category":"kube-audit","ccpNamespace":"5e4bf4baee195b00017cdbfa","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}","pod":"kube-apiserver-666bd4b459-hjgdc","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE","time":"2020-11-09T10:57:31.0000000Z"} diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json index c8aa29efa67..6a40062953c 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -8,7 +8,7 @@ "ingested":"2020-11-16T11:31:07.522993300Z", "kind":"event" }, - "message" : """{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe","stage":"ResponseComplete","requestURI":"/apis/admissionregistration.k8s.io/v1beta1?timeout=32s","verb":"get","user":{"username":"system:serviceaccount:kube-system:resourcequota-controller","uid":"33917550-525b-11ea-a4aa-168268435a05","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"]},"sourceIPs":["172.31.38.231"],"userAgent":"hyperkube/v1.14.8 (linux/amd64) kubernetes/ea670c3/system:serviceaccount:kube-system:resourcequota-controller","responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2020-11-09T10:57:31.643810Z","stageTimestamp":"2020-11-09T10:57:31.643954Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:discovery\" of ClusterRole \"system:discovery\" to Group \"system:authenticated\""}}""", + "message" : "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", "azure":{ "subscription_id":"70BD6E77-4B1E-4835-8896-DB77B8EEF364", "resource":{ From 1cbc280f9f0bb1af25a6403f95054550a85e4a5e Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 18 Nov 2020 11:15:18 +0100 Subject: [PATCH 16/19] map field --- filebeat/docs/fields.asciidoc | 10 ++++++++++ x-pack/filebeat/module/azure/fields.go | 2 +- .../module/azure/platformlogs/_meta/fields.yml | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index cec2daac3c6..dc28e51b7cf 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3148,6 +3148,16 @@ type: keyword Environment +type: keyword + +-- + +*`azure.platformlogs.EventTimeString`*:: ++ +-- +EventTimeString + + type: keyword -- diff --git a/x-pack/filebeat/module/azure/fields.go b/x-pack/filebeat/module/azure/fields.go index 9cdb45d510e..f37b4bf9ee8 100644 --- a/x-pack/filebeat/module/azure/fields.go +++ b/x-pack/filebeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzsXEtv4zgSvudXFHJsbGfvOSzgTacXAbY7jSR9FhiprHBCixqScuD+9QNS1ssiKTqm5AymdZqJPN/38VHFelDzGV5xdw3kVyXwAkBRxfAaLlf63y8vAAQyJBKv4RkVuQDIUKaClory4hr+cwEAYH4L33hWMQ2xpsgyeW1efYaCbLCD14/alXgNueBVuf+LBXMI04eS1XP764Rm7fsG+BV3b1z0/26Fr59aeh8S7r6MKFMuBDIShfGmw7JRKSxIoU5mqWFsBAIlr0SKI/z+gkygP4wxDlerTzkYjG9AE7R96v7Q+lyHo4hFN8ZtGEvBtzRDMQOpxvi3ZpElGcx2x25/uyh9PObvh2it+6jUCxf0V22CovYzkUhXfWwYYLf0qaJbqnaM53LSbg4dZidjTSqmEmMp17AmTGKYuX01tgVrLvb+qtEDWtBVsBVioajaWWfOZjYT83Znx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyHz80gLk6efGcEHLBc87cHC5ZIN/sw+l2vffEYLBZgSHMnK6xWIZLf/zUnUnslhGzqOHqBGzrhhbRs1XH1M7N+kLbsgCU2PnGVrd1SenlfHnPzBVltf1i2RKau9nyYaUJS3y/X9z+enyFOt1DmlwJszhPVYTBCEuQ6a8XMIwnDT9I8w6ishKVm6eRgrqcKkYRSxDMa51O0LKrZ/Ht3x9uYIzTIiUNC82WKjEt6QQOJlHjEI/D5whdBK8u+pAeIZrc/wdZi5Lqe74DyN2l+TlRR69Bc4yk731D5jJUtAipSVhi4v90TAfJ1MrOZfQEXcjj5co6pwjbqJz3+COY8penq6zBcu0nJbaVUy5x7sn1RuNqK5iE4/ZjtxGKURhzoU9RXkX7Y0NsTuItDHHJ73VuH7qUuitpSgexjfvTsd+uBCnEjKJYktTTAT+WaF0OLcp8wswu8eaBx5qHrgb47SSFFGVTFKe2VxCDC2GAAYE46y/yqgKSvmPyt016ihxnygO+PL6+Pv36QVbVOBrUC/YecIrgJtKCCwU2/0LVmY0VNa/KdgOZFWWXOhke0tYhVeL+tWnvlK3b+3otyjkOF6NpsAGH1SPeRe3tSLjKyyfOtZRkRk+zlkS38PqpW2t140/5W/reZnHr/2f5y78w1pmklFZMrKzWWAkNaumTLmnshfQDvaLQCI9+fxJgh4MtnHG2l+x6dlydn4iqnJ0hA6lMJ7ndbl0f2LPtoXyulLqohl7UUfsHkFNFyhbOZwdpkj0nuXo7IgoVNRjQ/oHJxqQxpeKbMYVkYkoINI8ONHb04WIHLXx1q0jX6XRVdmZoYL4ZFS1Lc9xLTGsiDfhKSFuBe2Lz1nC1KaPLMaRwrfL7k7aI4p4crG0M1EmJMsEStsKRxYDtISVh6zRVEkUSVfeWGbv/JQouprK9Bba8IyuKWZJF81YLReC6rKBrQJ4h7EfMQff9mNyR2jNE1oBLvAtMcmMe1PMUZ/6jm/gpz3CQc2ksXFWYTo5y84xkfcscwhsHUivG330wRUSRxRrLjb7GwaYE5HRIjcx6J6Zv7cnTUqXNUbro6ycFKEGtI8k24Krs8XdVx51CzTFpwDfCGPDWl7vZBAAwz1gKabNqm9VliHl/cOFX1rmeNknAhrH7RCIaU8/3RyhBqVlntGaQsMM+NuZ0tLNsgAboqUv3JxNmSXMDfmHtvzGiNJn3ke5MtfoOerK3O/2X2Tm3+2/foMrHuXjGK+d1LT8Hv+OsBO1ob1hvIrYahjDNUS3xZYKbu5FRFxCB2g7OsJYzBvfFryG6jElDH8WNOLo7JANYVN0HMVqp9yxtmOOezWj6oPndmJAueGYMsPx/fXRoafdGy2it4w17GdaxGwa/27A/vM6onv6HlR0AS7ss0Y1CzfaF45i5ulz157stEb3GZpyqUBTxSPu/vr7u3E3NbhBAGvHb1D7n7+/btLwoN76B2pjh/VFYs3ORJFiIGquiTFCfJ3kspyN21Gn61MvcA+kLMO2qbePF8NzlODC7z6UoTor1BNTSZzLVAyJnn2wknRmW2Tm7jxhCUlTlDKxZo6xZHV0UNPZE9VeHCdoTrW6uS+P3u+JQDS3R92bmsqEFgqFuaEx05a+k+DjaENM/opFQqWsUMxoYE+aBmoav4UNBM13bWcgyHtzpxRcbzSTmNENJpYPvhpFa8at53qAnh8tizm4gRawoYxRidrE3PtbUPmaZKgIZfNM1AOVr+AgGIhguEWWkDwXmOsoZEY5hgo8VBZhWSX0EtZh4+zaajYTpN6N2QbytAObaZMbQXb8w//ZwgKHbPvVftBJO3GMnNDqtpQi6yekm41CcOG69w9x78rcai6wcrU9KzRfRkw4gHNN1l7dEtfAvhiqqebpvuyi/cBOKtzMr+u+YQQPYyPvWfC3qfZuFFX/9RB98DuFSlTSWp+ZQdST5vJHB823SRNfM0b7Iqh9dfFXAAAA//+vNh7N" + return "eJzsXM9v2zoSvuevGORYbLP3HBbwpukiwLYpkvQsMNJY4Qst6pGUA/evfyBl/bJIio4pOQ+vPrWR830fh5zhcIbKZ3jF3TWQX5XACwBFFcNruFzp/19eAAhkSCRewzMqcgGQoUwFLRXlxTX85wIAwHwXvvGsYhpiTZFl8to8+gwF2WAHrz9qV+I15IJX5f4nFswhTB9KVs/ttxOatc8b4FfcvXHR/7kVvv7U0vuQcPdlRJlyIZCRKIw3HZaNSmFBCnUySw1jIxAoeSVSHOH3J2QC/WGMcThbfcrBYHwDmqDtU/eH1uc6HEUsujFuw1gKvqUZihlINca/NYssycDaHbv96aL08Zi/H6K14aNSL1zQX7ULijrORCJd9bFhgN3Sp4puqdoxnstJvzkMmJ2MNamYSoynXMOaMIlh7vbV+BasudjHq0YPaEFXwV6IhaJqZ7WczW0m7HZnx7NJ6MtIGaEbmdCCKkoUZsnzLqnkyH380gLk6c+N4YKWC5534OByyQb/Yh9Kta+/IwSDzQkOZeR0i8UyWv7npep2ZLGMnEcPUSNmXTG2jJqvPqbWNukLbsgCprHzDL3u6pPTy/jzH5gqy+P6QTIltfe1ZEPKkhb5/ncuP12e4r3OIQ32hDmix2qCICRkyJSXSziGk6a/hVlHEVnJys3TSEGdLhWjjGUoxjVvR0i59fP4pq8vV3CGCZGS5sUGC5X4phQCjXnEKPTngTOEToJ3VR0Iz3Bttr/Dk8tSqjv+w4zdJXl5kUcvgbNYsjf/AZYsBS1SWhK2uNgfDfNxMrWScwkdcTfyeImiPnPEPejcN7jjnLJ3TtenBYtZTjvaVUy5x7sn1QuNqK5iE4/ZjtxmKURhzoX9iPIu2hsbYrcRaWeOT3qrcf3UpdBLS1E8zG/efRz74UKcOpBJFFuaYiLwzwqlI7hNuV+A2z3WPPBQ88DdGKeVpIiqZJLyzBYSYmgxBDAgGJ/6q4yqoCP/UWd3jTo6uE8UB3zn+vjr9+kFW1Tga1Av2EXCK4CbSggsFNv9C1ZmNFTW3ynYDmRVllzow/aWsAqvFo2rT32l7tja0W9RyHG+Gk2BDT6oHvMubmtFxldYPnWsoyIzfJy9JH6E1VPbeq8bfyre1naZJ679n+cu/MNaZpJRWTKys3lgJDWrpky5p7IX0A7Wi0AiPef5kwQ9GGwTjHW8YtPWcnZ+IqpydIQOpTCe53W5dL9jz7aE8rpS6qIZR1FH7h5BTZcoWzmcHaZI9J7p6PyIKFTU40P6Cyc6kMaXimzGFZGJLCCSHZzo7e5CRI7aeevWka/S6KrszFBBfDKq2pbnuJYYVsSbiJQQt4L2xRcsYWrRRxbjOMK30+4+tEcU8eRiaS1RJiTLBErbDEcWA7SElYes0VRJFElX3lhm7fyUKLqayvQS2vCMrilmSZfNWD0Xguqyga0CeIezH2GDb/sxuTO05hNaAS7wLTGHGfeimKM+9R3fwE97RICaSWMTrMJ0cpadw5D3LHMIbANIrxt99MYVkkcUay42+xsGmBOR0SI3Oeiemb+3J01KlzdG66OsnBShDrTPJNuCq7PF3VcedQk0xaeA2Ahjx1pe72QSAMM1YCmmzapvVZYh5f3DiV9a5njaJxIax+0QiOlPP90coQ6lZZ7Rm0LTDPjbudLSzbIAH6KlL92cTZklzQ35R1t+Y0TpPe+jXJlr9Bx1Ze53+y8y8+/2X7/BFY/ycYzXGjUtv8e/I+xEbWhvGK8ithrGcA3RbbGlgpt7ERGn0AHakuopfqIbfFSCFnnkteMAbk1LGIt53dyC11A9poThz4JGNK0dsiFsKp6jRPGUC952zHGjaFT68FyNDKh1HFPjOL65P9pxdWylRfR+tYb9TIuYHevf3d9/Xjt2T9+Dii7AhX3WlGrhLv/CKdQ8TfY6kp3WZT9DRzAVaEqIxN3cf38r8KYGNwhgbTcOGg/zN/dNDSCosf+BeuhhTZlY1pmokAxEzWUYI8TXxi7L2bgdRcI+9QKXUMoybJl6m4gxIkcJLvzuLR2qj6TaMJXEuVzFkGjrg5Wkc9siMxf3CUtImqKUifXYGktWRwc1nf2U3MvjBM2pVjf3zdX7PRGI5uqqe1FTmdBCoTDXQ2Za0ncSfBxtislfsUiolBWKGR3sSdNATeP3sIGg+e4MDQR5rw2VguuFZg5mdIOJ5W2zRtGaceu+HqDnR8tiNm6gBWwoY1SidjH3+hZUviYZKkLZPIZ6oPIVHAQDEQy3yBKS5wJznYXMKMdQgYfKIiyrhJ7COm2cXVvNZpLUuzHbQJ4OYDMtciPIjn/4lx4W2GTbPxkQtNNObCMn9NktddD6E9JKRyG4cL10AHEv6txqLrBytQ0zNK9lTASAcxlrr26JO2hfDNVU53ZfdtFxYCcVbubXdd8wgoexkfcs+NtUbzmKqv96iD74hUYlKmmtz8wg6klz+bOD5sWoiVcpo72O1D66+CsAAP//n/8+qg==" } diff --git a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml index 14f27984dde..ac03e0004f5 100644 --- a/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml +++ b/x-pack/filebeat/module/azure/platformlogs/_meta/fields.yml @@ -41,6 +41,10 @@ type: keyword description: > Environment + - name: EventTimeString + type: keyword + description: > + EventTimeString - name: Caller type: keyword description: > From 9ce18710e1973e6390128dd51f7a159aaa8ce9d1 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 24 Nov 2020 16:51:48 +0100 Subject: [PATCH 17/19] update files --- .../test/platformlogs-eventhub.log-expected.json | 6 ++++-- .../platformlogs/test/platformlogs-kube.log-expected.json | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json index 10aa61d6db5..45ec7371ad1 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -7,9 +7,11 @@ "@timestamp" : "2020-11-03T09:06:42.000Z", "event" : { "action" : "Retreive ConsumerGroup", - "ingested" : "2020-11-05T12:51:52.507448200Z", + "dataset" : "azure.platformlogs", "kind" : "event", - "outcome" : "succeeded" + "outcome" : "succeeded", + "module": "azure", + "duration": 0 }, "azure" : { "subscription_id" : "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json index 6a40062953c..59a67eac603 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -5,8 +5,10 @@ "@timestamp":"2020-11-09T10:57:31.000Z", "event":{ "action":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "ingested":"2020-11-16T11:31:07.522993300Z", - "kind":"event" + "dataset" : "azure.platformlogs", + "kind":"event", + "module": "azure", + "duration": 0 }, "message" : "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", "azure":{ From 97b8e702827d234e241413a0bca1702c18319749 Mon Sep 17 00:00:00 2001 From: Mariana Date: Tue, 24 Nov 2020 19:42:49 +0100 Subject: [PATCH 18/19] fix logs --- .../platformlogs-eventhub.log-expected.json | 17 ++++- .../test/platformlogs-kube.log-expected.json | 74 ++++++++++++------- 2 files changed, 59 insertions(+), 32 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json index 45ec7371ad1..25fa8bc4bde 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -10,8 +10,19 @@ "dataset" : "azure.platformlogs", "kind" : "event", "outcome" : "succeeded", - "module": "azure", - "duration": 0 + "module": "azure" + }, + "fileset": { + "name": "platformlogs" + }, + "service": { + "type": "azure" + }, + "input": { + "type": "log" + }, + "log": { + "offset": 0 }, "azure" : { "subscription_id" : "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", @@ -38,8 +49,6 @@ } } }, - "input.type": "log", - "service.type": "azure", "tags": [ "forwarded" ] diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json index 59a67eac603..39b43a40212 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -1,36 +1,54 @@ +[ { - "cloud":{ - "provider":"azure" + "cloud": { + "provider": "azure" }, - "@timestamp":"2020-11-09T10:57:31.000Z", - "event":{ - "action":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "dataset" : "azure.platformlogs", - "kind":"event", - "module": "azure", - "duration": 0 + "@timestamp": "2020-11-09T10:57:31.000Z", + "fileset": { + "name": "platformlogs" }, - "message" : "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", - "azure":{ - "subscription_id":"70BD6E77-4B1E-4835-8896-DB77B8EEF364", - "resource":{ - "name":"OBSKUBE", - "id":"/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", - "provider":"MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", - "group":"OBS-INFRASTRUCTURE" + "service": { + "type": "azure" + }, + "input": { + "type": "log" + }, + "log": { + "offset": 0 + }, + "event": { + "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "dataset": "azure.platformlogs", + "kind": "event", + "module": "azure" + }, + "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", + "azure": { + "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "resource": { + "name": "OBSKUBE", + "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "group": "OBS-INFRASTRUCTURE" }, - "platformlogs":{ - "ccpNamespace":"5e4bf4baee195b00017cdbfa", - "operation_name":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "Cloud":"AzureCloud", - "Environment":"prod", - "category":"kube-audit", - "event_category":"Administrative", - "properties":{ - "pod":"kube-apiserver-666bd4b459-hjgdc", - "stream":"stdout" + "platformlogs": { + "ccpNamespace": "5e4bf4baee195b00017cdbfa", + "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "Cloud": "AzureCloud", + "Environment": "prod", + "category": "kube-audit", + "event_category": "Administrative", + "properties": { + "pod": "kube-apiserver-666bd4b459-hjgdc", + "stream": "stdout" } } - } + }, + "tags": [ + "forwarded" + ] } + ] + + From b9dc88dc9e7567cd1436192c7d54e82dece7df84 Mon Sep 17 00:00:00 2001 From: Mariana Date: Wed, 25 Nov 2020 09:20:08 +0100 Subject: [PATCH 19/19] generate tests --- .../platformlogs-eventhub.log-expected.json | 78 +++++++---------- .../test/platformlogs-kube.log-expected.json | 83 +++++++------------ 2 files changed, 59 insertions(+), 102 deletions(-) diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json index 25fa8bc4bde..ca2c95be824 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-eventhub.log-expected.json @@ -1,54 +1,34 @@ [ { - "cloud" : { - "region" : "West Europe", - "provider" : "azure" - }, - "@timestamp" : "2020-11-03T09:06:42.000Z", - "event" : { - "action" : "Retreive ConsumerGroup", - "dataset" : "azure.platformlogs", - "kind" : "event", - "outcome" : "succeeded", - "module": "azure" - }, - "fileset": { - "name": "platformlogs" - }, - "service": { - "type": "azure" - }, - "input": { - "type": "log" - }, - "log": { - "offset": 0 - }, - "azure" : { - "subscription_id" : "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", - "resource" : { - "name" : "OBSTESTEVENTHUBS", - "id" : "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", - "provider" : "MICROSOFT.EVENTHUB/NAMESPACES", - "group" : "OBS-TEST" - }, - "platformlogs" : { - "status" : "Succeeded", - "Caller" : "Portal", - "ActivityId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8", - "EventTimeString" : "11/3/2020 9:06:42 AM +00:00", - "Environment" : "PROD", - "category" : "OperationalLogs", - "event_category" : "Administrative", - "ScaleUnit" : "PROD-AM3-AZ501", - "properties" : { - "SubscriptionId" : "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", - "TrackingId" : "30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2", - "Namespace" : "obstesteventhubs", - "Via" : "sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04&$skip=0&$top=100" - } - } - }, + "@timestamp": "2020-11-03T09:06:42.000Z", + "azure.platformlogs.ActivityId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8", + "azure.platformlogs.Caller": "Portal", + "azure.platformlogs.Environment": "PROD", + "azure.platformlogs.EventTimeString": "11/3/2020 9:06:42 AM +00:00", + "azure.platformlogs.ScaleUnit": "PROD-AM3-AZ501", + "azure.platformlogs.category": "OperationalLogs", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.properties.Namespace": "obstesteventhubs", + "azure.platformlogs.properties.SubscriptionId": "7657426d-c4c3-44ac-88a2-3b2cd59e6dba", + "azure.platformlogs.properties.TrackingId": "30ed877c-a36b-491a-bd4d-ddd847fe55b8_M2CH3_M2CH3_G3S2", + "azure.platformlogs.properties.Via": "sb://obstesteventhubs.servicebus.windows.net/insights-logs-operationallogs/consumergroups?api-version=2017-04&$skip=0&$top=100", + "azure.platformlogs.status": "Succeeded", + "azure.resource.group": "OBS-TEST", + "azure.resource.id": "/SUBSCRIPTIONS/7657426D-C4C3-44AC-88A2-3B2CD59E6DBA/RESOURCEGROUPS/OBS-TEST/PROVIDERS/MICROSOFT.EVENTHUB/NAMESPACES/OBSTESTEVENTHUBS", + "azure.resource.name": "OBSTESTEVENTHUBS", + "azure.resource.provider": "MICROSOFT.EVENTHUB/NAMESPACES", + "azure.subscription_id": "7657426D-C4C3-44AC-88A2-3B2CD59E6DBA", + "cloud.provider": "azure", + "cloud.region": "West Europe", + "event.action": "Retreive ConsumerGroup", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "event.outcome": "succeeded", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "service.type": "azure", "tags": [ "forwarded" ] diff --git a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json index 39b43a40212..fb95fe0ba80 100644 --- a/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json +++ b/x-pack/filebeat/module/azure/platformlogs/test/platformlogs-kube.log-expected.json @@ -1,54 +1,31 @@ [ -{ - "cloud": { - "provider": "azure" - }, - "@timestamp": "2020-11-09T10:57:31.000Z", - "fileset": { - "name": "platformlogs" - }, - "service": { - "type": "azure" - }, - "input": { - "type": "log" - }, - "log": { - "offset": 0 - }, - "event": { - "action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "dataset": "azure.platformlogs", - "kind": "event", - "module": "azure" - }, - "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", - "azure": { - "subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", - "resource": { - "name": "OBSKUBE", - "id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", - "provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", - "group": "OBS-INFRASTRUCTURE" - }, - "platformlogs": { - "ccpNamespace": "5e4bf4baee195b00017cdbfa", - "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", - "Cloud": "AzureCloud", - "Environment": "prod", - "category": "kube-audit", - "event_category": "Administrative", - "properties": { - "pod": "kube-apiserver-666bd4b459-hjgdc", - "stream": "stdout" - } - } - }, - "tags": [ - "forwarded" - ] -} - ] - - - + { + "@timestamp": "2020-11-09T10:57:31.000Z", + "azure.platformlogs.Cloud": "AzureCloud", + "azure.platformlogs.Environment": "prod", + "azure.platformlogs.category": "kube-audit", + "azure.platformlogs.ccpNamespace": "5e4bf4baee195b00017cdbfa", + "azure.platformlogs.event_category": "Administrative", + "azure.platformlogs.operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "azure.platformlogs.properties.pod": "kube-apiserver-666bd4b459-hjgdc", + "azure.platformlogs.properties.stream": "stdout", + "azure.resource.group": "OBS-INFRASTRUCTURE", + "azure.resource.id": "/SUBSCRIPTIONS/70BD6E77-4B1E-4835-8896-DB77B8EEF364/RESOURCEGROUPS/OBS-INFRASTRUCTURE/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/OBSKUBE", + "azure.resource.name": "OBSKUBE", + "azure.resource.provider": "MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS", + "azure.subscription_id": "70BD6E77-4B1E-4835-8896-DB77B8EEF364", + "cloud.provider": "azure", + "event.action": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "event.dataset": "azure.platformlogs", + "event.kind": "event", + "event.module": "azure", + "fileset.name": "platformlogs", + "input.type": "log", + "log.offset": 0, + "message": "{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"22af12c3-a1fe-4f2c-99a9-3cdde671dbfe\"}", + "service.type": "azure", + "tags": [ + "forwarded" + ] + } +]