From 5aed15128e7e8d548cd3b6a8e7231ad86ce9efbf Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 5 Mar 2020 17:42:10 +0100 Subject: [PATCH 01/11] Make CEF key name mapping case-insensitive There's some case inconsistency in CEF docs (i.e. C6a4Label). Better to ignore case when mapping keys to full names. --- x-pack/filebeat/module/cef/log/test/checkpoint.log | 0 .../module/cef/log/test/checkpoint.log-expected.json | 0 x-pack/filebeat/processors/decode_cef/cef/cef.go | 2 +- x-pack/filebeat/processors/decode_cef/cef/keys.go | 10 ++++++++++ .../decode_cef/testdata/samples.log.golden.json | 2 +- 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 x-pack/filebeat/module/cef/log/test/checkpoint.log create mode 100644 x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log b/x-pack/filebeat/module/cef/log/test/checkpoint.log new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/x-pack/filebeat/processors/decode_cef/cef/cef.go b/x-pack/filebeat/processors/decode_cef/cef/cef.go index c9827baeedd..72d94bc969d 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/cef.go +++ b/x-pack/filebeat/processors/decode_cef/cef/cef.go @@ -124,7 +124,7 @@ func (e *Event) Unpack(data string, opts ...Option) error { } for key, field := range e.Extensions { - mapping, found := extensionMapping[key] + mapping, found := extensionMappingLowerCase[strings.ToLower(key)] if !found { continue } diff --git a/x-pack/filebeat/processors/decode_cef/cef/keys.go b/x-pack/filebeat/processors/decode_cef/cef/keys.go index f747c27fbd6..52907ed1d82 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/keys.go +++ b/x-pack/filebeat/processors/decode_cef/cef/keys.go @@ -4,6 +4,8 @@ package cef +import "strings" + type mappedField struct { Target string Type DataType @@ -657,3 +659,11 @@ var extensionMapping = map[string]mappedField{ Type: TimestampType, }, } + +var extensionMappingLowerCase = map[string]mappedField{} + +func init() { + for k, v := range extensionMapping { + extensionMappingLowerCase[strings.ToLower(k)] = v + } +} diff --git a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json index 89a6edb2d94..c5ca7c5a93f 100644 --- a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json +++ b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json @@ -948,7 +948,6 @@ "agentType": "agent_ac", "agentVersion": "7.1.7.7602.0", "assetCriticality": "0", - "c6a4Label": "Agent IPv6 Address", "categoryBehavior": "/Access/Stop", "categoryDeviceGroup": "/Firewall", "categoryDeviceType": "Firewall", @@ -958,6 +957,7 @@ "deviceAddress": "111.111.111.99", "deviceAssetId": "5Wa8hHVSDFBCc-t56wI7mTw==", "deviceCustomIPv6Address4": "ffff:0:0:0:222:5555:ffff:5555", + "deviceCustomIPv6Address4Label": "Agent IPv6 Address", "deviceCustomNumber1Label": "ICMP Type", "deviceCustomNumber2Label": "ICMP Code", "deviceCustomNumber3Label": "DurationInSeconds", From 9639d3a8e7301b32db709dd110d040c754b1ce6c Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 5 Mar 2020 19:07:14 +0100 Subject: [PATCH 02/11] Add missing custom CEF extensions This adds: - `deviceCustomIPv6Address2(Label)`: Only 1, 3 and 4 were expected. - `flexNumber[12](Label)`: These two alternative custom numbers were dropped after V23 of the spec, but still used by some vendors. [Maybe unnecessary] changes: - Changed the case of `DeviceCustomNumber2` from uppercase as documented) to lowercase to align with the other fields. --- filebeat/docs/fields.asciidoc | 66 +++++++++++++++++-- .../processors/decode_cef/_meta/fields.yml | 36 ++++++++-- .../processors/decode_cef/cef/keys.go | 52 +++++++++++---- .../filebeat/processors/decode_cef/fields.go | 2 +- 4 files changed, 132 insertions(+), 24 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 38670881f3d..d3425983b30 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3656,6 +3656,15 @@ type: double -- +*`cef.extensions.deviceCustomIPv6Address1`*:: ++ +-- +One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. + +type: ip + +-- + *`cef.extensions.deviceCustomIPv6Address1Label`*:: + -- @@ -3665,7 +3674,7 @@ type: keyword -- -*`cef.extensions.deviceCustomIPv6Address4`*:: +*`cef.extensions.deviceCustomIPv6Address2`*:: + -- One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. @@ -3674,7 +3683,7 @@ type: ip -- -*`cef.extensions.deviceCustomIPv6Address4Label`*:: +*`cef.extensions.deviceCustomIPv6Address2Label`*:: + -- All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. @@ -3683,7 +3692,7 @@ type: keyword -- -*`cef.extensions.deviceCustomIPv6Address1`*:: +*`cef.extensions.deviceCustomIPv6Address3`*:: + -- One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. @@ -3692,7 +3701,16 @@ type: ip -- -*`cef.extensions.deviceCustomIPv6Address3`*:: +*`cef.extensions.deviceCustomIPv6Address3Label`*:: ++ +-- +All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + +type: keyword + +-- + +*`cef.extensions.deviceCustomIPv6Address4`*:: + -- One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. @@ -3701,7 +3719,7 @@ type: ip -- -*`cef.extensions.deviceCustomIPv6Address3Label`*:: +*`cef.extensions.deviceCustomIPv6Address4Label`*:: + -- All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. @@ -3728,7 +3746,7 @@ type: keyword -- -*`cef.extensions.DeviceCustomNumber2`*:: +*`cef.extensions.deviceCustomNumber2`*:: + -- One of three number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. @@ -3917,6 +3935,42 @@ type: keyword -- +*`cef.extensions.deviceFlexNumber1`*:: ++ +-- +One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. + +type: long + +-- + +*`cef.extensions.deviceFlexNumber1Label`*:: ++ +-- +All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + +type: keyword + +-- + +*`cef.extensions.deviceFlexNumber2`*:: ++ +-- +One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. + +type: long + +-- + +*`cef.extensions.deviceFlexNumber2Label`*:: ++ +-- +All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + +type: keyword + +-- + *`cef.extensions.deviceHostName`*:: + -- diff --git a/x-pack/filebeat/processors/decode_cef/_meta/fields.yml b/x-pack/filebeat/processors/decode_cef/_meta/fields.yml index 3deb1c2a951..64b0a32cd02 100644 --- a/x-pack/filebeat/processors/decode_cef/_meta/fields.yml +++ b/x-pack/filebeat/processors/decode_cef/_meta/fields.yml @@ -274,27 +274,35 @@ type: double description: One of four floating point fields available to map fields that do not apply to any other in this dictionary. + - name: deviceCustomIPv6Address1 + type: ip + description: One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. + - name: deviceCustomIPv6Address1Label type: keyword description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. - - name: deviceCustomIPv6Address4 + - name: deviceCustomIPv6Address2 type: ip description: One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. - - name: deviceCustomIPv6Address4Label + - name: deviceCustomIPv6Address2Label type: keyword description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. - - name: deviceCustomIPv6Address1 + - name: deviceCustomIPv6Address3 type: ip description: One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. - - name: deviceCustomIPv6Address3 + - name: deviceCustomIPv6Address3Label + type: keyword + description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + + - name: deviceCustomIPv6Address4 type: ip description: One of four IPv6 address fields available to map fields that do not apply to any other in this dictionary. - - name: deviceCustomIPv6Address3Label + - name: deviceCustomIPv6Address4Label type: keyword description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. @@ -306,7 +314,7 @@ type: keyword description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. - - name: DeviceCustomNumber2 + - name: deviceCustomNumber2 type: long description: One of three number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. @@ -390,6 +398,22 @@ type: keyword description: The facility generating this event. For example, Syslog has an explicit facility associated with every event. + - name: deviceFlexNumber1 + type: long + description: One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. + + - name: deviceFlexNumber1Label + type: keyword + description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + + - name: deviceFlexNumber2 + type: long + description: One of two alternative number fields available to map fields that do not apply to any other in this dictionary. Use sparingly and seek a more specific, dictionary supplied field when possible. + + - name: deviceFlexNumber2Label + type: keyword + description: All custom fields have a corresponding label field. Each of these fields is a string and describes the purpose of the custom field. + - name: deviceHostName type: keyword description: The format should be a fully qualified domain name (FQDN) associated with the device node, when a node is available. diff --git a/x-pack/filebeat/processors/decode_cef/cef/keys.go b/x-pack/filebeat/processors/decode_cef/cef/keys.go index 52907ed1d82..232d9dcbf9f 100644 --- a/x-pack/filebeat/processors/decode_cef/cef/keys.go +++ b/x-pack/filebeat/processors/decode_cef/cef/keys.go @@ -12,8 +12,13 @@ type mappedField struct { } // extensionMapping is a mapping of CEF key names to full field names and data -// types. This mapping was generated from tables contained in "Micro Focus -// Security ArcSight Common Event Format Version 25" dated September 28, 2017. +// types. This mapping was generated from tables contained in: +// - "Micro Focus Security ArcSight Common Event Format Version 25" +// dated September 28, 2017. +// - "Check Point Log Exporter CEF Field Mappings" +// dated November 23, 2018. +// - "HPE Security ArcSight Common Event Format Version 23" +// dated May 16, 2016. var extensionMapping = map[string]mappedField{ "agt": { Target: "agentAddress", @@ -235,22 +240,22 @@ var extensionMapping = map[string]mappedField{ Target: "deviceCustomFloatingPoint4", Type: FloatType, }, + "c6a1": { + Target: "deviceCustomIPv6Address1", + Type: IPType, + }, "c6a1Label": { Target: "deviceCustomIPv6Address1Label", Type: StringType, }, - "c6a4": { - Target: "deviceCustomIPv6Address4", + "c6a2": { + Target: "deviceCustomIPv6Address2", Type: IPType, }, - "C6a4Label": { - Target: "deviceCustomIPv6Address4Label", + "c6a2Label": { + Target: "deviceCustomIPv6Address2Label", Type: StringType, }, - "c6a1": { - Target: "deviceCustomIPv6Address1", - Type: IPType, - }, "c6a3": { Target: "deviceCustomIPv6Address3", Type: IPType, @@ -259,6 +264,14 @@ var extensionMapping = map[string]mappedField{ Target: "deviceCustomIPv6Address3Label", Type: StringType, }, + "c6a4": { + Target: "deviceCustomIPv6Address4", + Type: IPType, + }, + "C6a4Label": { + Target: "deviceCustomIPv6Address4Label", + Type: StringType, + }, "cn1": { Target: "deviceCustomNumber1", Type: LongType, @@ -268,7 +281,7 @@ var extensionMapping = map[string]mappedField{ Type: StringType, }, "cn2": { - Target: "DeviceCustomNumber2", + Target: "deviceCustomNumber2", Type: LongType, }, "cn2Label": { @@ -443,6 +456,23 @@ var extensionMapping = map[string]mappedField{ Target: "fileModificationTime", Type: TimestampType, }, + "flexNumber1": { + Target: "deviceFlexNumber1", + Type: LongType, + }, + "flexNumber1Label": { + Target: "deviceFlexNumber1Label", + Type: StringType, + }, + "flexNumber2": { + Target: "deviceFlexNumber2", + Type: LongType, + }, + "flexNumber2Label": { + Target: "deviceFlexNumber2Label", + Type: StringType, + }, + "fname": { Target: "filename", Type: StringType, diff --git a/x-pack/filebeat/processors/decode_cef/fields.go b/x-pack/filebeat/processors/decode_cef/fields.go index 30d9e1abba1..d885d6210e1 100644 --- a/x-pack/filebeat/processors/decode_cef/fields.go +++ b/x-pack/filebeat/processors/decode_cef/fields.go @@ -19,5 +19,5 @@ func init() { // AssetDecodeCef returns asset data. // This is the base64 encoded gzipped contents of processors/decode_cef. func AssetDecodeCef() string { - return "eJzsPV1z20aS7/4VXboHO1WUIvlrb3WVvdKKVswry+ZZUnJ1dVXJEGiScxrMIDMDUsiv35ovECAJkAJAJ7aiB5dJAv09Pd2NnsYx3GN+DhFOnwFoqhmewxAjESNcvruCVIoIlRISphRZrJ4BxKgiSVNNBT+HfzwDALgUSSI4vFsg13AlZEI0vLh8d/UdxESTk2fg7z63Vx8DJwkGnOZP5ymew0yKLPXfbEFi/v6ZQ4xTkjENeo7wa2wp/SXC6a8lUpeSalRAGLP4YSpFYq+/fHdVgEpQKTJD0AL0nCr41QIRk//HSJ/ASEMkuCaUq3AnzJHEGAQBhMfmlwIePmjkigpe8Gz+ynyXeV+gNNcW3wcZ3GO+FDIufV8jCfP3kwMCYlrQqFKM6JRGxFwPmcIYJrn91fN78myDlhgXNMKTBfJYyK4UGRiBIAcY9Jxoo504izDejxZ3te5GzNgB6U7NIZTlOWxDDppl9kvEiFK/0LgbVXec/pYh0Bi5plOKhe4sEgtxCyEKFyipzvfAjQ8kSY1T+Qllfvyezub7ETZKUiE14RFWKDqB2znCgjAag9KS8pn5kJnVLhHu+D0XSz6AD2I5qIC7xphmyQAMAQO7dgt6yiAp1zhDWYZ5evzqhw1wr4/f/hBA/u0Y/v2HFdy/H5+d/rACvik88283pd3MhdTlC6oi2kRZ+Ca1gbjsdHegvRSMYRTw3WN+bMUEKaFSQUSkpGhEWDijlUu0bvCkgsZ68V/sD+cwJUyVhbLuNsvMkBlyfRHHEpWqXBBYouna1xWejLJHYyAOQJDchYxu6GxuvT7HSAtZrEyzqfiluS7fDbKGXA1FQijfStimkrdSN/x4A7GFYkH3TOJ7ofTHqgk+msK5UPoApI3WET+KKAsCRsOeibomUZOx7Unc9cXlgUzuo+5kcQ2QP2OENNW3tMZYYqLXf9jgW9MEgWhYzmk0B8qnNjg0LoFMRKZLW82SKJAG42IVs2yKqEkShtD/FbyTZTsjslT/LnjfBn4rCVeMaIxb+6+9oBspvHvQKDlho2H/llHFdPd5dAAUedqHIvO0jQ4b6PppIxZ8NGmLaiTYk20dWucH0XSaMp+rjKXQIhKsrWAvVqCA4QKZkZ+FOAgxaDmqe397Ox7Yf28GcHPzfvFyALfIOOoBjD+NBzC6Hl+Yfy/MBSauUwIEr1HAhCi0ye+lyLjeygITfNZIP0TmXiBKiYiapQVLqucuN/Wh73uxhITw3PonZT2m/VmZrdh5UTFRKBcY/ydYUiAiHCYIIqHagKRToBqogrM6TnKNarTdvHex8DFLJi6BsFBAGy8xRSltVDgRGY8HIJERTRc+70ZQIpOR/RSj0pQ7DbqrBFdzmg4gQcJNpG+Xhc3oDetTJpbmW5vdbwXTxOOnrJ2empgUmbZctmDyBK6EDIY6sDcZ+MALdK6SUebZJbUreBVkNbxHmdIiQXkgTxHA9+0oSmy2j/xHIclVG6ILoZm1sFVMInGKUhnBUg6Em7yBo14Kee+SRhfNmPVkf1y8DoBqpF9C2XemkBJZVDoiYaxII0wzxnL4LSPMsB1XsooXV/89/PjdbkJ/RPGBaKqzuCYKFNmE7YwDmQNBOWHODTsTdmHhmjaeq3KCNkGzJPeQqCFU8FlnSj2MQ5LaNQdrsGRrwYQ/zoDVXGQsNhsF2cNmtuxRVRq4iHEAyzlyIPaDXSELQhmZsDrPVALQPeW6oQ8QCSb4scIUpSV1jg8kxogmhHm3uoemOmZXVsg/Ux6LpdqW02/xQbtpGgvZbu9a1blSIXUQgo2HJqiXiBxObazz9s2bV2/2IMRFqDVp+y5qxlIsaOxNeJW2l0XiQ+CtBudjosq2Sacr0y/K9wHIaAhnp28GcHR2+ubIWKQvw/of92a3a/GkrH9L6nO1jend9NygXNAIu9KjHBjQRM5Qh+S7iDp3ktFDUrvu0HQBctO37bU7P4Lo1qvJ3AhkqlGamNoG4wXQ/zCedWWXBKZU4pIwdgI/bV+Ap4aPPdfdF0n0a/G1j+2sud19HlnhGD2uAIOBvFJwxS8qhRrmxOyvaD/SGTdZsdV7yJ53S+1OoWxfYWzYcjOF0iya0XDdHXG4+zj6HxfOSyG0u5QqmCFHScxGu+7a7BWjIZzux9CBoojAksFpIgWqgsO0v9T646orO4F3CaEs7Gw+9RZTjRwSkqY2L/RZS2DGl/5deCIxoik1a90EECaZjWlMtM2k0kyHu6kKDxj2EdlY0gVlOMNOBV2dpzQKEaJj7OgiTiinSkuihTwawJFBd+TKB0c/Zqj0URBms+ifm10rUHkCo2BHVacSabqgOgd8wCgzuhB8zdKWIawr8HmFrYkEVjIxO9MaI7vF+mXcUM/O52Aex+ymF/Y5WetilnvKpsk98lAL9w9/G5H2lxrbiKCSFe+TUzQRd2lLA1dMEE35bCwo12cfyATbV/wY8/WG0JIxJwuTwURCSlSp4DHlM2AGh/cP8I5Ecx97qeBqnG/xD5PNUnVYJl4WaSZToYqIrYzxsfy+emL8vv72+R0SjWetno598k+XlsKVczVJ0qK3KOTKZpUlpPjeFUAFcKFt7Tw3vxOeg9BzE1hwtxnG1PoPIvN9OXgainr51Svq5bevqOoO0bKQ53U2FZmEqQcIqYH4R2iuwlKNEX7VLD0xq3z17anw9TfB0mi8eOuD4CewpZW43a6+5pi/rDoDqgj3/1jFPYGgsWym347itnvFr5OVb98GXR/DdvPbVfkOEfFcYtGh0LvC4E4hqJQY7lnuWnAQ74FAIiQWBx0GpXtAZWnKKMYOq3sCmQqlaNNjxw2ZfJvKH24wuj0SfdrKfwKhrGN0u7d+2sp/Am7/xoLd7vb34NHrX9EHT+BXrncvjqei9+0e/8nq/Qk4e8fodmf/ZPX+ZPz89rLAk9X7EygtOEbf/KX3ijieit7f/qX3iji+Zb0PqcT6JpOdx414vuU05tLqNgC29IQDRRCJJMl4OGI1J8q1p4Q+cuZPppT6oYwOhdQYwzEcnR7Zxht/DgiEhKMz91U4NdPM7Z/8uISh0Z0AIxpnQuZt6fyMqUSFXDvriDy4VROSbwcSks5svxKfhdYgX+JRvsMuU2iupBLEkgdA9HenPxXNMSFm6do5EnSah4bqd/6w3NH314JTLeT3Q6ruv/+MJK7vBLPshwaw1j2WF07U1sNkdjAFyzdb5Wxrkmug1O5U2M5eZXPLFYkoq86seLQBTT2MGvTV7s+bXDExsyuFcMCHlNGI6hWM9SZKXKDM92CkjwP8fZ77sPpoeeTD3DtyLmHENcopidq3swYAIHjptE5KonvUxuHYE3RorvJnavdoqev5KAp5/FEU2+J/6FMo5Xa/Rko+eWf9hbTFcKr3VdWY5EyQuL372ZyFE1pFUwe64RxKI2GHOCYTDq74bXqbW3wEbV38iQfhLKrhpM4FLw5iJ/a8+wQrnKzRrpz7RK5lHjrpG1npf1xF+aiJO6Xgm9WL1uvy0Ir1I6HX19cQx5DneQ7v358nyblRl4SEMkYVRoLHChTlEQKmIprDi/8iHM6UhrO//+30uyZW+5h0YVi1Iy6Ckbc1oUOf/9lsRe50BKhK8kE71zdRfZmzM05i3ZvYDy6c7iJpbunvLgjkcc8OpfAdaz7FGTTyuEdPYh1JxfMWJ0IkmtysWOUmJZuaNBWVqp+cYGlsuY+FQ0TEh/ZmL7OqKuafOM3Y1Ywmj25yPfa3T5mORPuNa0hVykjuXI5wsAaQqcwfzILnKovM1vTcCPv5lFCWSXxeR1Hn9OfWbfBhaCPhW5O82znmNr0OhFIeSSTKXONjycG23ILrushuShleSiQa25u6MXMb/RtZGoB2a4ws1LrM3lz2nqh5W3mZe53VGkgNSDpkpNxoZF2aDiFEYS0ZnqlJehpouBZxMZezbzEzojQkFkGjsHmHKO9jKV+wiAVnObww8hCZBqoVpETP6yok5pYx0a11fZUxZhEEd2kADozts8yW4SxNNgSlWiGbNtGBMqGqyxSnFQRVlkkDzhv6+3bR73Kb5sY9cXQameUHZVkxvkhpigNQwuRiA0AdndSEo1OGD+2PwFysH6jYLDqT0iW+8oyqrvYc45TanX0Nal1NmjBmOQiF2j5r1MZPr0rAxl/7E6PM7Py+pmriljAyx9brLF5XlDE8ThBM7DLJwQ3nVZBxZnMtNPtSfV9joZdOFXA3uKUobz+ukL0SbAON/bSjfJHu896fYvzxFtJPU8hf0u8i/a9jhXbrlDkslX5edocQj8gJ1dKYTBgKP6MLQ5y1rRg1oUytDyw9geuMaXrMKEdbpaKowri9Yp73JIfMRub/ZxJPezPHJdh7fDm4dqapYPFVn5G5YPG+kblH3UdwbnMrj7kZW99Rehnz/rG6J6bncL0i+33CdU9Gl7psOWLfTwO9BugBZTlIL5KH3YF6IOkQsfp+0uglZN8PVR+Re8Xc94/gJVna59a9Vvc+I1HdZuRKCwEIB5LFVJdmRPv6NMaVx61wNCExpEQpA/3I9hdk7m0AdhRKGLzi/ABhStinnhxQSiHN5RJ1JjlExjWsHoKfPpy9fPW67sG3xN8yVPqSUeS6NIG2U41XoTy+sNOLt3k0j3IHQYJrfGirUxiuPqx6JLg2JK2NQvT4ikIVxvCiMqTm/e3tGD6jHZAq60wwEC3uafuhPP729kK7Rj0XnYp3ltnEgnFVPBPVRm6CHdx9/tCM/062DrBG3DeLuLCJcEeKBzwojUiqvtXm7vMHT2AoOptvfIDrrwzP6OwsZRPELJGxGk7c6NneHgOFsbndxs84KH/uBiJHY+dRq7uAdxuP2gS9a1PKP9Z+hM62sA6v37YXT0657WUDY7LRBhP2FXUOz+dCafdgwfzvxOE9iURS94DBoWzVkFK8hOfo9PT8ND5/e3pOpudnk/O3Z0fNkUyX3hVH8KF6V8IzP6+J5uYVd9EhBqfunNvoUXfqAbmttH54jncPR92DoEMMM63S10hF3yNM5+6BqWvPtYUeayeP6RZ0hB22qaHszDo0M6yT2tq+L8p27WeaTtCKa0X0JF8fQNjDVNN1Fg7YcrAN1Vc1y9Qx0OsYU2+I5Qmmew379PdV38x28NmnKwH0OPZ0TQRu4um2uaU2adxrbKlCHjteW88sXbH6B48rHen1DuySwFrOKfUt7HvMKd1vEqmj6OCu46DzR0MY08lBaCJ76H8sGgwKza3tUdIRZNF9ob5Huw+Z7aXre4LWVv8HkqM8fr1KcLMiFQ6PisprJlymQGXR3GTDt5djw9zdcFyjFF1X29u1KZ/a994o+3IhJ/0BnPnvyGwmcWY85wBe+u/ssST3Rhm3hl+Fi+0TMe/ZfRWg+logY5IrNApeGArhdD1b/bdVr5Z3d1s5DmdW3HGULrXNoX9la56Wcic4hrEUD/kARsObAfyMEzBhJMrt8g/UfLJv9m1Libt7PWKj/qlQtRRJi9cuEQ4iDZGnyjUmA9vXbmQ98NVx1FHdu3o85f/EOVnQyqt5H0W7nyYsJBCYeFibPU08bOMjE8MvfSTtosDYuipf13dvSG4m+Rajue3ya+0sAwBPgS2nvcCT2Ql8PxQ3dXWUqun9uPZu00dR8KMLU4q2aAMLGL1HuApRbzMNN3TG7TMk3v6UxOWcSBJpLM5t+YiL1rwdd8cS6NYv6W+vviHY6URlJtUbgO+RHFhj0xqTVNepKiGczFB27dr/OWxWFzJS1jW9u7levUhyXTTVHf3EJ44npc64beKoRAiBjJu1pLMuLq4iLo/H7xu7e4PHGvZ/BQAA//9XpFa2" + return "eJzsPWlz28aS3/0rurQf7FRRiuTr7dNW3paeaMXcsmyujmRra6uSIdAkZzWYQWYGpJBf/2ouEOABUgDoxGL0wQlJoM+Znu5Gd+MYHjA/hwjHLwA01QzPoY+RiBEuP1xBKkWESgkJY4osVi8AYlSRpKmmgp/DP14AAFyKJBEcPsyQa7gSMiEaXl1+uPoOYqLJyQvwd5/bq4+BkwQDTvOn8xTPYSJFlvpv1iAxf//MIcYxyZgGPUX4NbaU/hLh+NcSqXNJNSogjFn8MJYisddffrgqQCWoFJkgaAF6ShX8aoGI0f9jpE9goCESXBPKVbgTpkhiDIIAwmPzSwEPHzVyRQUveDZ/Zb7LvM9QmmuL74MMHjCfCxmXvt8gCfP3kwMCYlzQqFKM6JhGxFwPmcIYRrn91fN78mKFlhhnNMKTGfJYyLYUGRiBIAcY9JRoo504izDejRZ3tW5HzNABaU/NPpTlOWxCDppt9kvEiFK/0LgdVfec/pYh0Bi5pmOKhe4sEgtxDSEKZyipznfAjY8kSY1R+QllfvyRTqa7ETZIUiE14RFWKDqBuynCjDAag9KS8on5kJndLhHu+QMXc96DT2Leq4C7xphmSQ8MAT27dwt6yiAp1zhBWYZ5evzmhxVwb4/f/xBA/u0Y/v2HBdy/H5+d/rAAvio88287pd1OhdTlC6oiWkVZ2Ca1grhsdLegvRSMYRTwPWB+bMUEKaFSQUSkpGhEWBijhUm0ZvCkgsZa8V/sD+cwJkyVhbJsNsvMkAlyfRHHEpWqXBBYounS1xWejLIHQyAOQJDchYxu6WRqrT7HSAtZ7ExzqPituSzfFbL6XPVFQihfS9iqktdS1/98C7GFYkF3TOJHofTn6hJ8MoVTofQeSBssI34SURYEDPodE3VNorrFtiNx1xeXe1pyn3WrFVcD+QYjpKm+oxsWS0z08g8rfGuaIBAN8ymNpkD52DqHxiSQkch06aiZEwXSYJwtfJZVEdVJwhD6v4K3WtluEVmqfxe86wV+JwlXjGiMG9uvnaAbKXx41Cg5YYN+9yujiun+ZrAHFHnahSLztIkOa+j6acUXfDJps6on2NHa2rfO96LpNGU+VhlKoUUkWFPBXixAAcMZMiM/C7EXfNCyV/fx7m7Ys//e9uD29uPsdQ/ukHHUPRh+GfZgcD28MP9emAuMX6cECL5BASOi0Aa/lyLjei0LTPBJLf0QmXuBKCUiarYWzKmeutjUu74fxRwSwnNrn5S1mPZnZY5iZ0XFSKGcYfyfYEmBiHAYIYiEagOSjoFqoArONnGSa1SD9ct7Gwufs2TkAggLBbSxEmOU0nqFI5HxuAcSGdF05uNuBCUyGdlPMSpNudOgu0pwNaVpDxIk3Hj6dlvYiN6wPmZibr610f1aMHU8fsma6amOSZFpy2UDJk/gSsiwUHv2JgMfeIHOZTLKPLugdgGvgmwD71GmtEhQ7slSBPBdG4oSm809/0EIctWK6IJrZlfYwieROEapjGApB8JN3MBRz4V8cEGj82bMfrI/zt4GQBukX0LZdaSQEllkOiJhVpFGGGeM5fBbRphhO65EFa+u/rv/+bvthP6I4hPRVGfxBi9QZCO21Q9kDgTlhDkz7JawcwuXtPFSlQO0EZotuYNEDaGCT1pT6mHsk9S2MVjNSrYrmPCnLWA1FRmLzUFBdlgza86oKg1cxNiD+RQ5EPvB7pAZoYyM2CbLVALQPuS6pY8QCSb4scIUpSV1io8kxogmhHmzuoOmWkZXVsg/Ux6LuVoX06+xQdtpGgrZ7Oxa5LlSIXUQgvWHRqjniBxOra/z/t27N+92IMR5qBvC9m3UDKWY0dgv4UXYXhaJd4HXLjjvE1WOTTpeLP0ifR+ADPpwdvquB0dnp++OzIr0aVj/487stk2elPVvSX2p1jG9nZ5blDMaYVt6lAMDmsgJ6hB8F17nVjI6CGqXDZouQK7atp1O5ycQ3Xg3mRuBjDVK41NbZ7wA+h/Gsi7WJYExlTgnjJ3AT+s34KnhY8d991UC/Y34mvt2drnd3wyscIweF4DBQF4ouGIXlUINU2LOV7Qf6YSbqNjqPUTP26V2r1A2zzDWHLmZQmk2zaC/bI443H8e/I9z56UQ2l1KFUyQoyTmoF02bfaKQR9Od2NoT15EYMngNJ4CVcFg2l822uOqKTuBDwmhLJxsPvQWY40cEpKmNi70UUtgxqf+nXsiMaIpNXvdOBAmmI1pTLSNpNJMh7upCg8YdhHZUNIZZTjBVgldnac0Ch6iY+zoIk4op0pLooU86sGRQXfk0gdHP2ao9FEQZr3oX5pTK1B5AoOwjqpGJdJ0RnUO+IhRZnQh+NJKmwe3rsDnFbYkEljIxJxMS4xsF+vXMUMdG5+9WRxzml7Y52SNk1nuKZsmD8hDLtw//K1F2l1obD2CSlS8S0xRR9ylTQ1cMUE05ZOhoFyffSIjbJ7xY8znG0JJxpTMTAQTCSlRpYLHlE+AGRzePsAHEk2976WCqXG2xT9MNlvVYRl5WaSZTIUqPLYyxqfy++bA+H37/PntE41njZ6OffFPl+bCpXM1SdKitijEymaXJaT43iVABXChbe48N78TnoPQU+NYcHcYxtTaDyLzXTk4DEW9/uYV9fr5K6p6QjRM5HmdjUUmYewBQmog/hGaq7C0YRF+0ywd2Kp88/xU+PZZsDQYzt57J3i95aj3gsvMGFCFA/wHs/L8t1eJ2/X28ZtU3AHYxRK3663iN6m4AwjUStyut/3fpOIOIOJ0BSjrj7dtjyxCKDOVWJSWdK4wuFcIKiWGe5a72inEByCQCIlFh0qvdA+oLE0ZxdhhdY+OU6EUrXtevCKTQ1H++iPysJV/AGetY3T9MXvYyj+A8/rWgl1v9nfg0etf0UdP4Deudy+OQ9H7eot/sHo/AGPvGF1v7A9W7wdj59eHZAer9wMI6xyj7/7Se0Uch6L393/pvSKO56z3PpW4uTpoa58Yz9e00c6tbgNgS0/oBINIJEnGQ2/clChXVxQaAJhvKSoVshkdCqkxhmM4Oj2yFVO+gQuEhKMz91Vod6rn9k/e52JodK17RONEyLwpnTeYSlTItVsdkQe3qB7zdVxC0oktNOOTUNMFfftf5UsjM4XmSipBzHkARH93+lPRFBNitq4dAELHeaiE/+C7HI++vxacaiG/71P18P0NknhzCZ9lP1TuNS6OvXCithYmsxNFWL5a42hrylzlq3btfFuLzM0tVySirDps5MkLaOxhbEBfLdu9zRUTE7tTCAd8TBmNqF7AWK5+xRnKfBdGGD52kb2dCyDMqsz2GD6LbE5JNs/Z9i/YbJfDfd5L4FmH913MX+mybc9a5YYde+begXMMBlyjHJOoeTdCAACCl5otUxI9oDZuh22ARnOVH4mwQ0V0x52E5OmdhLZDa99NhOVq7VpKvniX7Stpi+FY76qqIcmZIHFzJ2R1lFmo9E8d6Jo2wlrC9tHlGPoOvbO+zjl6Am1t7IkH4VZUTaPlBS/maCR2XMkIK5ws0a6cE4Vcyzw0QtWy0v20oXKnoGsy871GRedMeebQckf/9fU1xDHkeZ7Dx4/nSXJu1CUhoYxRhZHgsQJFeYSAqYim8Oq/CIczpeHs7387/a6O1S4GFRlW7YSisMibLqF9t2+udpK06uCskrzXxqNVVF+n9dFJrH0P0t6F014k9R1Z7QWBPO7YoBS2Y8mmuAWNPO7QklhDUrG8RUOfxFTIxS7nsTlYCChUavPgG0tjw3Ms9IASH+Cbs8yqqhhf5TRjdzMad7rO9NjfvmQ6Es0Prj5VKSO5MznCwepBpjLfVwsvVRaZo+mlEfbLMaEsk/hyE0WtkyB37oAPM3cJX5vquZtibpNsgVDKI4lEmWu8L9lbl2HgepNnN6YMLyUSjc2Xulnm1vs3sjQA7dEYWaibwhlz2Ueipk3lZe51q9ZAqkHSIi/FjUaWpekQQhT2kuGZmqCnhoZrERdjlbsWMyNKQ2IR1Aqbt/DyPpfiBYtYcJbDKyMPkWmgWkFK9HRTntTcMiS6sa6vMsYsgmAuDcCeWfsss9G4pcm6oFQrZOM6OlAmVLUZwreAoMoyqcF5S39fL/ptZtPcuCOOVhMP/ZxDK8ZXKU2xB0qYWKwHqKOTDe7omOFj8w7Gi+V+uNXMEyld4tNPqDYloGIcU3uyL0HdlJgijFkOQr6my0SVsdOLTJCx177hn5mT3z9ZMX5LmHhms/YWr0vKGB5HCMZ3GeXgZqsryDizsRaac2lzdXOhl1aJMDd3q8hyPS2ftRBsDY3dFKV9leahzlOZf/wK6aY07C/pt5H+t7FD2yXU90ulf91BCxePyBHV0iyZ8E6PCZ0Z4uzailETytTyvOkTuM6YpseMcrRZKooqTEstXscwyiGznvn/mcDT3sxxDvYenw7eOJJasPiqS89csHhXz9yj7sI5t7GVx1yPrWsvvYx5d1/dE9Oxu16R/S7uuiejTV627LHvpoFOHfSAsuykF8HDdkc9kLQPX303aXTisu+GqgvPvbLcd/fgJZnb6pVOs3s3SFS7EefSQgDCgWQx1aUR/z4/jXGl6AKORiSGlChloB/ZKqPMvczFTrIKc7OcHSBMCfvUkwNKKaS5XKLOJIfImIZFKczp49nrN283lb9I/C1DpS8ZRa5LA8Rb5XgVyuMLO3x+nUXzKLcQJLjGx6Y6hf7iw6JSimtD0tIkW4+vSFRhDK8qM8Y+3t0N4QbtfGu5aQkGosUDbT5Tzd/eXGjXqKeiVfLOMptYMC6LZ7zayA0ghfubT/X472VjB2vAfcmYc5sId6R4wL3ShLvqS8nubz55AkPS2XzjHVx/ZXhGZ0fhGydmjoxt4MRNDu/sMVCYet5uepiD8ucuI3Q0tp6UvQ14u+nWddDbFqX8Y+lHaL0WluF1W/biySmXvaxgTFbKYMK5os7h5VQo7R4smP87cXhPIpFsesDgUDYqSCneoXZ0enp+Gp+/Pz0n4/Oz0fn7s6N6T6ZN7YojeF+1K+GZn9dEffGKu2gfc6+3jt31qFvVgNxVSj88x9tnW+9A0D5mUVfpq6Wi6wnUU/fA1FXp2USPXSdPqRl2hO23qKFszFoUMyyT2nh9X5TXtR9JPUIrrgXRo3x5fmwHQ6mXWdhjycE6VN/UKGrHQKdTqP1CLA+g3mlWs7+v+mLNvY+uXgigw6nVSyLgdmD1urHTNmjcaeq0Qh47XhuPnF6w+gdPmx7o5T6MksAajpn2jSw7jJnebZC0o2jvpmOv46ODG9PKQGgiO6h/LAoMCs0tnVHSEWTRfaW6R3sOmeOl7Wvelnb/J5KjPH67CHCzIhQOj4rKeyZcpkBl0dREw3eXQ8PcfX+4QSl6U25v26F8al9bpuy74Zz0e3DmvyOTicSJsZw9eO2/s90J7oVgbg+/CRfbJ2LesvssQPWtbmZJLtAoeGUohNPlaPXfFrVa3tyt5Th0rrmmtDa5zb5/43aelmInOIahFI95Dwb92x78jCMwbiTK9fIP1HyxL2ZvSom7e9ljo/6pUDUVSYu35hEOIg2ep8o1Jj1b125k3fPZcdTRhpUTKP8nTsmMVt6s/iTa/TB4IYHAyMNarWni4RgfGB9+7j1p5wXG1lT5vL57wX09yXcYTW2VX2NjGQB4Cmw67RWeTE7g+7643ZRHqS69H5deTf0kCn50bkpRFm1gAaMPCFfB662n4ZZOuH2GxJt3SVxOiSSRxqJ703tcdMPLzbdsgXb1kv726gvenU5UZkK9HvgayZ5dbFpjkupNqkoIJxOUbav2fw6H1YWMlDVNH26vF+8BXhZN9UQ/8YHjSakybp04Kh5CION2Kejc5BdXEZffbtI1dvcCpiXs/woAAP//C9svNQ==" } From b2210e7b3b1f7ce5377b70d64015283be9d32861 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 5 Mar 2020 22:35:31 +0100 Subject: [PATCH 03/11] CEF module: Support Check Point devices This adds a new ingest pipeline and fields to populate from Check Point CEF logs. Closes #16041 --- filebeat/docs/fields.asciidoc | 554 ++++++++++++++++++ x-pack/filebeat/module/cef/fields.go | 2 +- .../filebeat/module/cef/log/_meta/fields.yml | 250 ++++++++ .../module/cef/log/ingest/cp-pipeline.yml | 306 ++++++++++ .../module/cef/log/ingest/fp-pipeline.yml | 2 +- .../module/cef/log/ingest/pipeline.yml | 3 + x-pack/filebeat/module/cef/log/manifest.yml | 1 + .../module/cef/log/test/checkpoint.log | 3 + .../cef/log/test/checkpoint.log-expected.json | 182 ++++++ 9 files changed, 1301 insertions(+), 2 deletions(-) create mode 100644 x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d3425983b30..e37e30f96eb 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -4754,6 +4754,560 @@ type: keyword -- +[float] +=== checkpoint + +Fields for Check Point custom string mappings. + + + +*`checkpoint.app_risk`*:: ++ +-- +Application risk. + +type: keyword + +-- + +*`checkpoint.app_severity`*:: ++ +-- +Application threat severity. + +type: keyword + +-- + +*`checkpoint.app_sig_id`*:: ++ +-- +The signature ID which the application was detected by. + +type: keyword + +-- + +*`checkpoint.auth_method`*:: ++ +-- +Password authentication protocol used. + +type: keyword + +-- + +*`checkpoint.category`*:: ++ +-- +Category. + +type: keyword + +-- + +*`checkpoint.confidence_level`*:: ++ +-- +Confidence level determined. + +type: keyword + +-- + +*`checkpoint.connectivity_state`*:: ++ +-- +Connectivity state. + +type: keyword + +-- + +*`checkpoint.cookie`*:: ++ +-- +IKE cookie. + +type: keyword + +-- + +*`checkpoint.dst_phone_number`*:: ++ +-- +Destination IP-Phone. + +type: keyword + +-- + +*`checkpoint.email_control`*:: ++ +-- +Engine name. + +type: keyword + +-- + +*`checkpoint.email_id`*:: ++ +-- +Internal email ID. + +type: keyword + +-- + +*`checkpoint.email_recipients_num`*:: ++ +-- +Number of recipients. + +type: long + +-- + +*`checkpoint.email_session_id`*:: ++ +-- +Internal email session ID. + +type: keyword + +-- + +*`checkpoint.email_spool_id`*:: ++ +-- +Internal email spool ID. + +type: keyword + +-- + +*`checkpoint.email_subject`*:: ++ +-- +Email subject. + +type: keyword + +-- + +*`checkpoint.event_count`*:: ++ +-- +Number of events associated with the log. + +type: long + +-- + +*`checkpoint.file_hash`*:: ++ +-- +File hash (SHA1 or MD5). + +type: keyword + +-- + +*`checkpoint.frequency`*:: ++ +-- +Scan frequency. + +type: keyword + +-- + +*`checkpoint.icmp_type`*:: ++ +-- +ICMP type. + +type: long + +-- + +*`checkpoint.icmp_code`*:: ++ +-- +ICMP code. + +type: long + +-- + +*`checkpoint.identity_type`*:: ++ +-- +Identity type. + +type: keyword + +-- + +*`checkpoint.incident_extension`*:: ++ +-- +Format of original data. + +type: keyword + +-- + +*`checkpoint.integrity_av_invoke_type`*:: ++ +-- +Scan invoke type. + +type: keyword + +-- + +*`checkpoint.peer_gateway`*:: ++ +-- +Main IP of the peer Security Gateway. + +type: ip + +-- + +*`checkpoint.performance_impact`*:: ++ +-- +Protection performance impact. + +type: keyword + +-- + +*`checkpoint.protection_id`*:: ++ +-- +Protection malware ID. + +type: keyword + +-- + +*`checkpoint.protection_name`*:: ++ +-- +Specific signature name of the attack. + +type: keyword + +-- + +*`checkpoint.protection_type`*:: ++ +-- +Type of protection used to detect the attack. + +type: keyword + +-- + +*`checkpoint.scan_result`*:: ++ +-- +Scan result. + +type: keyword + +-- + +*`checkpoint.sensor_mode`*:: ++ +-- +Sensor mode. + +type: keyword + +-- + +*`checkpoint.severity`*:: ++ +-- +Threat severity. + +type: keyword + +-- + +*`checkpoint.malware_status`*:: ++ +-- +Malware status. + +type: keyword + +-- + +*`checkpoint.subscription_expiration`*:: ++ +-- +The expiration date of the subscription. + +type: date + +-- + +*`checkpoint.tcp_flags`*:: ++ +-- +TCP packet flags. + +type: keyword + +-- + +*`checkpoint.termination_reason`*:: ++ +-- +Termination reason. + +type: keyword + +-- + +*`checkpoint.update_status`*:: ++ +-- +Update status. + +type: keyword + +-- + +*`checkpoint.user_status`*:: ++ +-- +User response. + +type: keyword + +-- + +*`checkpoint.uuid`*:: ++ +-- +External ID. + +type: keyword + +-- + +*`checkpoint.virus_name`*:: ++ +-- +Virus name. + +type: keyword + +-- + +*`checkpoint.malware_name`*:: ++ +-- +Malware name. + +type: keyword + +-- + +*`checkpoint.malware_family`*:: ++ +-- +Malware family. + +type: keyword + +-- + +*`checkpoint.voip_log_type`*:: ++ +-- +VoIP log types. + +type: keyword + +-- + +[float] +=== cef.extensions + +Extra vendor-specific extensions. + + + +*`cef.extensions.cp_app_risk`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.cp_severity`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.ifname`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.inzone`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.layer_uuid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.layer_name`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.logid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.loguid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.match_id`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.nat_addtnl_rulenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.nat_rulenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.origin`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.originsicname`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.outzone`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.parent_rule`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.product`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.rule_action`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.rule_uid`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.sequencenum`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.service_id`*:: ++ +-- +type: keyword + +-- + +*`cef.extensions.version`*:: ++ +-- +type: keyword + +-- + +*`observer.ingress.zone`*:: ++ +-- +-- + +*`observer.egress.zone`*:: ++ +-- +-- + +*`observer.interface.name`*:: ++ +-- +-- + [[exported-fields-cisco]] == Cisco fields diff --git a/x-pack/filebeat/module/cef/fields.go b/x-pack/filebeat/module/cef/fields.go index ce9bfda5de2..f4d5d5f2d46 100644 --- a/x-pack/filebeat/module/cef/fields.go +++ b/x-pack/filebeat/module/cef/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCef returns asset data. // This is the base64 encoded gzipped contents of module/cef. func AssetCef() string { - return "eJx8kMFq8zAQhO9+inmB5AF0+C/5a+ihp5RejZFWzhJZK7Syi9++yImDk0L3JHZGsx9zwJUWA0v+MIqbAjVA4RLI4PTWNoAjtZlTYYkG/xoA+FiN8JKRyRLPHIfqRpBBITNlnBcNMhzxeSHcctE7p5gpOslriiay7NnCMwWn4FgtXA+hCMqFNqU+HVlx1FnySFksqd5jUpaZHemxwd1v1n2dA2I/kqmklpJwLA8JKEsigyHLlHZbR76fQunWKAPfB6Un+Vcb27Q32NpK+7iH06RFRpxLriWNfUocB919fGXec8+cJ+3YPYkb+ZWWb8mv2h+Adb5qIt7/Nz8BAAD//0k3k/4=" + return "eJy8Wc9v6zYMvvevILDLdmiAd3iXHAY8pO0WDN0L0O5dBVWmHS62qEm02+yvHyTnV9u0Vu1iOQWR9H0fKVIilUvY4HYOBsvLhou2xgsAIalxDovrmwuAAoPx5ITYzuHXCwCA2zQRSvbg0SB1ZKs4G2quAnCHHu62oeZqBvdrhB4XdFEE6NAW7BNKcGioJAMlYV0EIBunUCQCYZA17kfi1wINF6gMluA8GwxhB+M8d1RgmF3Abv48/R4/l2B1g/Oo1KBjsnIYApCtwzlUnlt38muBpW5rUQlqDqWuAz4bfuWN/eemFxu9cnPgg0UbhBu4Ex+d1GjnyFbhZOFLzae6O/JtUFQ8G9wr3+D2kf3LsXcExs+PiAjLq8PA6y97crNGs/kfnbaIfLBKXjO918Jzr80y3aadU57CZqzbvjlXk9EpEiPO7LWTXtIF7NCTbD+DUtYetcAeMoedqglREnM0UGW1tB5heQWPazLrlHX6RNajDlCgoBEs4GFQVitr1aCsebSulQ4hzklYaGUvxHkWNlxDG7AYkGG0YMV+9L4sduuHaNiWVKA1qGrssB5Nd8CBhJMc7huyw4aytWiEOpKtCqIFJ2g4IEFCGqTmDY2mW/5xvUMYoCmCKLdmi8q2zQP6sYRXGIRsH0rL1eUqQg5QY6OpVoateB69t9e2IosJMYtufEIvraC3uu6BYHmVxefRkCO0EqKDz3LXbKv3iP9MGwNcwhEsiztgCMT282zeAebaHhzzJ3o8wWVztw9/o5HRYdUz9iBDhB1aUYZbe54uf38TUgAdAhvS8UZ4JOnvjFj3vS+jpBrVWof1WJtvqEaIAPDz3e/fvgB7uL36+ssQrcd/WrRm9GVwZ7Q9ogywkWmcitCjPL1c3K7S1ByWWBmPZ4mrh1iKeP3K9m17cnJkB5JlljWJU+GToI2ZPDpU2DdaYsiyp4pijhZa9CC/YBWLL6U7RbbjDU6yPUVOj5NjvkP0qtKCj/p8sJJ7j+1WU7zaotExISMa3KFpo0HwWw87qMCX0XOxoqHG6fHn08pzrBlT4XYEhR50SMVh7YSz+URAo+tHnWrcfOL4w+ht37e4x/I6wu03RotoM9RdnEiZEoH3W5d4j3CpfI6ddl/U5ysKRlvlMbT16KBI+dBDDJGhDexV89YRl0OWIKAZPuemdnH3H+rcdtGYqvU2jOW83cV0jzJkYPtwWKrwyZFPdfBZ7uJ1D/GqbTxCpOn7wD6lGVAkxqmy1tVo++8XK3DabFAg4QzxpXYqSVYedRh/s9wfkaBHGqBuXfTRxO3+K4Hk7XYb0E+lC+hjojq2YSh72nb8IX39tCugB4/m/k1syqncv4Fl9GH7BJ1Ctk/PD9CVuqF69Bm0J+xRhpzJ5FTN1aSr5QcvV7HqTyvOxeThoQDL2aGgOw3Jz35VvH4Sr3dvzZeHd+Yj9XsPiWf9ZJz60KPiWyAfumDOV6ZldjS+Udn+y3bC+lpv0avsVH8HY5odNVeTJHA1yYRGi1nnFqVnEawWpYtCbK18W+NbTy7ZWJNB+v5o6vpAZtrGcivTItRpHztH3/+XNRbEc9FmNj1nASK90ubN6iofZFKchv6xYlpkBPQdxUZwgpAOfXYbf/jyE9x/v/p+KHqE4XpxB19mX0HbAjw23J1cqocIeoiC0c/IVh5DmD2Lp1ezMGcSWUFfaoOzFN3/BQAA//9c6pxr" } diff --git a/x-pack/filebeat/module/cef/log/_meta/fields.yml b/x-pack/filebeat/module/cef/log/_meta/fields.yml index 0d24bf8458f..16e54d71cf1 100644 --- a/x-pack/filebeat/module/cef/log/_meta/fields.yml +++ b/x-pack/filebeat/module/cef/log/_meta/fields.yml @@ -8,3 +8,253 @@ type: keyword description: > Virus ID + + +- name: checkpoint + type: group + default_field: false + description: > + Fields for Check Point custom string mappings. + fields: + - name: app_risk + type: keyword + description: Application risk. + + - name: app_severity + type: keyword + description: Application threat severity. + + - name: app_sig_id + type: keyword + description: The signature ID which the application was detected by. + + - name: auth_method + type: keyword + description: Password authentication protocol used. + + - name: category + type: keyword + description: Category. + + - name: confidence_level + type: keyword + description: Confidence level determined. + + - name: connectivity_state + type: keyword + description: Connectivity state. + + - name: cookie + type: keyword + description: IKE cookie. + + - name: dst_phone_number + type: keyword + description: Destination IP-Phone. + + - name: email_control + type: keyword + description: Engine name. + + - name: email_id + type: keyword + description: Internal email ID. + + - name: email_recipients_num + type: long + description: Number of recipients. + + - name: email_session_id + type: keyword + description: Internal email session ID. + + - name: email_spool_id + type: keyword + description: Internal email spool ID. + + - name: email_subject + type: keyword + description: Email subject. + + - name: event_count + type: long + description: Number of events associated with the log. + + - name: file_hash + type: keyword + description: File hash (SHA1 or MD5). + + - name: frequency + type: keyword + description: Scan frequency. + + - name: icmp_type + type: long + description: ICMP type. + + - name: icmp_code + type: long + description: ICMP code. + + - name: identity_type + type: keyword + description: Identity type. + + - name: incident_extension + type: keyword + description: Format of original data. + + - name: integrity_av_invoke_type + type: keyword + description: Scan invoke type. + + - name: peer_gateway + type: ip + description: Main IP of the peer Security Gateway. + + - name: performance_impact + type: keyword + description: Protection performance impact. + + - name: protection_id + type: keyword + description: Protection malware ID. + + - name: protection_name + type: keyword + description: Specific signature name of the attack. + + - name: protection_type + type: keyword + description: Type of protection used to detect the attack. + + - name: scan_result + type: keyword + description: Scan result. + + - name: sensor_mode + type: keyword + description: Sensor mode. + + - name: severity + type: keyword + description: Threat severity. + + - name: malware_status + type: keyword + description: Malware status. + + - name: subscription_expiration + type: date + description: The expiration date of the subscription. + + - name: tcp_flags + type: keyword + description: TCP packet flags. + + - name: termination_reason + type: keyword + description: Termination reason. + + - name: update_status + type: keyword + description: Update status. + + - name: user_status + type: keyword + description: User response. + + - name: uuid + type: keyword + description: External ID. + + - name: virus_name + type: keyword + description: Virus name. + + - name: malware_name + type: keyword + description: Malware name. + + - name: malware_family + type: keyword + description: Malware family. + + - name: voip_log_type + type: keyword + description: VoIP log types. + +- name: cef.extensions + type: group + default_field: false + description: > + Extra vendor-specific extensions. + fields: + + - name: cp_app_risk + type: keyword + + - name: cp_severity + type: keyword + + - name: ifname + type: keyword + + - name: inzone + type: keyword + + - name: layer_uuid + type: keyword + + - name: layer_name + type: keyword + + - name: logid + type: keyword + + - name: loguid + type: keyword + + - name: match_id + type: keyword + + - name: nat_addtnl_rulenum + type: keyword + + - name: nat_rulenum + type: keyword + + - name: origin + type: keyword + + - name: originsicname + type: keyword + + - name: outzone + type: keyword + + - name: parent_rule + type: keyword + + - name: product + type: keyword + + - name: rule_action + type: keyword + + - name: rule_uid + type: keyword + + - name: sequencenum + type: keyword + + - name: service_id + type: keyword + + - name: version + type: keyword + +# TODO: Update to ECS 1.5 and remove. +- name: observer.ingress.zone +- name: observer.egress.zone +- name: observer.interface.name diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml new file mode 100644 index 00000000000..1cf4399dbb5 --- /dev/null +++ b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml @@ -0,0 +1,306 @@ +--- +description: Pipeline for Check Point CEF + +processors: + # This script is mapping CEF extensions to ECS when possible. Otherwise + # it maps them to fields under the `checkpoint` group using Check Point log + # field names. + # + # [1] Description of Check Point CEF extensions: + # https://community.checkpoint.com/t5/Logging-and-Reporting/Log-Exporter-CEF-Field-Mappings/td-p/41060 + # [2] Description of Check Point log field names (sk144192): + # https://supportcenter.checkpoint.com/supportcenter/portal?eventSubmit_doGoviewsolutiondetails=&solutionid=sk144192 + # + # Note that in some cases the CEF extension name doesn't accurately describe + # its contents. For example sntdom/sourceNtDomain, which is used to store + # Check Point's domain_name, documented as "Domain name sent to DNS request". + # + # This script processes the `params.extensions` list below. This list consists + # of two different kinds of mappings, the simpler has a source ext `name` + # and a `to` field. It copies the given extension field to the target `to`. + # + # When the `labels` dict is defined, the target field depends on the value of + # the accompanying label field. For example, the field deviceCustomIPv6Address2 + # is mapped to `source.ip` only when the extension deviceCustomIPv6Address2Label + # exists and its value is "Source IPv6 Address". + # + # Also it can convert the destination value by simple mapping when the + # convert key exists. Values without an entry in the convert dict are not + # copied and the target field remains unset. + # + # The output of this processor is a single field, `_tmp_copy`, that contains + # a list of actions `{"to": "target_field", "value":"field value"}` that is + # later executed using a foreach processor. This is done to avoid complex + # de-dotting and other gotchas of setting arbitrary fields in Painless. + - script: + lang: painless + params: + extensions: + - name: cp_app_risk + to: checkpoint.app_risk + + - name: cp_app_risk + to: event.risk_score + # This mapping is a mix of [1] and [2] above. + convert: + unknown: 0 + informational: 0 + very-low: 1 + low: 2 + medium: 3 + high: 4 + very-high: 5 + critical: 5 + + - name: cp_severity + to: checkpoint.severity + + - name: cp_severity + to: event.severity + convert: + # This mapping is a mix of [1] and [2] above. + unknown: 0 + informational: 0 + very-low: 1 + low: 1 + medium: 2 + high: 3 + very-high: 4 + critical: 4 + + # Number of events associated with the log + - name: baseEventCount + to: checkpoint.event_count + + # Log type + - name: deviceExternalId + to: observer.type + + # Product Family + - name: deviceFacility + to: observer.type + convert: + '0': Network + '1': Endpoint + '2': Access + '3': Threat + '4': Mobile + + # Gateway interface, where the connection is received from in case of an outbound connection + - name: deviceInboundInterface + to: observer.ingress.interface.name + + # Gateway interface, where the connection is sent from, in case of an inbound connection + - name: deviceOutboundInterface + to: observer.egress.interface.name + + - name: externalId + to: checkpoint.uuid + + - name: fileHash + to: checkpoint.file_hash + + - name: reason + to: checkpoint.termination_reason + + # Possibly an IKE cookie + - name: checkrequestCookies + to: checkpoint.cookie + + # Domain name sent to DNS request + - name: sourceNtDomain + to: dns.question.name + + # CVE registry entry + - name: Signature + to: vulnerability.id + + - name: Recipient + to: destination.user.email + + - name: Sender + to: source.user.email + + - name: deviceCustomFloatingPoint1 + labels: + update version: observer.version + + - name: deviceCustomIPv6Address2 + labels: + source ipv6 address: source.ip + + - name: deviceCustomIPv6Address3 + labels: + destination ipv6 address: destination.ip + + - name: deviceCustomNumber1 + labels: + payload: network.bytes + elapsed time in seconds: host.uptime + email recipients number: checkpoint.email_recipients_num + + - name: deviceCustomNumber2 + labels: + duration in seconds: event.duration + icmp type: checkpoint.icmp_type + + - name: deviceCustomNumber3 + labels: + icmp code: checkpoint.icmp_code + + - name: deviceCustomString1 + labels: + application rule name: rule.name + dlp rule name: rule.name + threat prevention rule name: rule.name + connectivity state: checkpoint.connectivity_state + email id: checkpoint.email_id + voip log type: checkpoint.voip_log_type + + - name: deviceCustomString2 + labels: + # Protection malware id + protection id: checkpoint.protection_id + update status: checkpoint.update_status + email subject: checkpoint.email_subject + sensor mode: checkpoint.sensor_mode + scan invoke type: checkpoint.integrity_av_invoke_type + category: checkpoint.category + # Matched categories + categories: rule.category + peer gateway: checkpoint.peer_gateway + + - name: deviceCustomString6 + labels: + application name: process.name + virus name: checkpoint.virus_name + malware name: checkpoint.malware_name + malware family: checkpoint.malware_family + + - name: deviceCustomString3 + labels: + user group: group.name + # Format of original data. + incident extension: checkpoint.incident_extension + identity type: checkpoint.identity_type + email spool id: checkpoint.email_spool_id + # Type of protection used to detect the attack + protection type: checkpoint.protection_type + + - name: deviceCustomString4 + labels: + malware status: checkpoint.spyware_status + destination os: os.name + scan result: checkpoint.scan_result + frequency: checkpoint.frequency + protection name: checkpoint.protection_name + user response: checkpoint.user_status + email control: checkpoint.email_control + tcp flags: checkpoint.tcp_flags + threat prevention rule id: rule.id + + - name: deviceCustomString5 + labels: + matched category: rule.category + authentication method: checkpoint.auth_method + email session id: checkpoint.email_session_id + vlan id: network.vlan.id + + - name: deviceCustomDate2 + labels: + subscription expiration: checkpoint.subscription_expiration + + - name: deviceFlexNumber1 + labels: + confidence: checkpoint.confidence_level + + - name: deviceFlexNumber2 + labels: + destination phone number: checkpoint.dst_phone_number + performance impact: checkpoint.performance_impact + + - name: flexString1 + labels: + application signature id: checkpoint.app_sig_id + + - name: flexString2 + labels: + malware action: event.action + attack information: event.action + + - name: rule_uid + to: rule.uuid + + - name: ifname + to: observer.interface.name + + - name: inzone + to: observer.ingress.zone + + - name: outzone + to: observer.egress.zone + + - name: product + to: observer.product + + source: | + def actions = new ArrayList(); + def exts = ctx.cef?.extensions; + if (exts == null) return; + for (entry in params.extensions) { + def value = exts[entry.name]; + if (value == null || + (entry.convert != null && + (value=entry.convert[value.toLowerCase()]) == null)) + continue; + if (entry.to != null) { + actions.add([ + "value": value, + "to": entry.to + ]); + continue; + } + def label = exts[entry.name + "Label"]; + if (label == null) continue; + def dest = entry.labels[label.toLowerCase()]; + if (dest == null) continue; + actions.add([ + "value": value, + "to": dest + ]); + } + ctx["_tmp_copy"] = actions; + + - foreach: + field: _tmp_copy + processor: + set: + field: "{{_ingest._value.to}}" + value: "{{_ingest._value.value}}" + + - remove: + field: _tmp_copy + + # event.duration is a string and contains seconds. Convert to long nanos. + - script: + params: + second_to_nanos: 1000000000 + lang: painless + source: | + def duration = ctx.event?.duration; + if (duration == null) return; + ctx.event.duration = Long.parseLong(duration) * params.second_to_nanos; + on_failure: + - remove: + field: event.duration + ignore_missing: true + + # checkpoint.file_hash can be either MD5 or SHA1. + - set: + field: 'file.hash.md5' + value: '{{checkpoint.file_hash}}' + if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==32' + - set: + field: 'file.hash.sha1' + value: '{{checkpoint.file_hash}}' + if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==40' diff --git a/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml index 3fe032c00fb..1459d521cce 100644 --- a/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/fp-pipeline.yml @@ -24,4 +24,4 @@ processors: - set: field: cef.forcepoint.virus_id value: "{{cef.extensions.deviceCustomString4}}" - if: "ctx.cef?.extensions?.deviceCustomString4 != null" + if: "ctx.cef?.extensions?.deviceCustomString4 != null" diff --git a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml index 2d14926e7f8..75a86ea2758 100644 --- a/x-pack/filebeat/module/cef/log/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/pipeline.yml @@ -80,6 +80,9 @@ processors: - pipeline: name: '{< IngestPipeline "fp-pipeline" >}' if: "ctx.cef?.device?.vendor == 'FORCEPOINT'" + - pipeline: + name: '{< IngestPipeline "cp-pipeline" >}' + if: "ctx.cef?.device?.vendor == 'Check Point'" on_failure: - set: field: error.message diff --git a/x-pack/filebeat/module/cef/log/manifest.yml b/x-pack/filebeat/module/cef/log/manifest.yml index 670a3188a4e..60115d99b40 100644 --- a/x-pack/filebeat/module/cef/log/manifest.yml +++ b/x-pack/filebeat/module/cef/log/manifest.yml @@ -16,6 +16,7 @@ var: ingest_pipeline: - ingest/pipeline.yml - ingest/fp-pipeline.yml + - ingest/cp-pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log b/x-pack/filebeat/module/cef/log/test/checkpoint.log index e69de29bb2d..8951c3edade 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log @@ -0,0 +1,3 @@ +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\=R80,O\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration +CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json index e69de29bb2d..e9753166af1 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json @@ -0,0 +1,182 @@ +[ + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.destinationAddress": "52.173.84.157", + "cef.extensions.destinationPort": 443, + "cef.extensions.destinationTranslatedAddress": "0.0.0.0", + "cef.extensions.destinationTranslatedPort": 0, + "cef.extensions.deviceAction": "Accept", + "cef.extensions.deviceCustomDate2": "2017-10-16T10:42:13.713Z", + "cef.extensions.deviceCustomDate2Label": "This field is made up", + "cef.extensions.deviceCustomString2Label": "Rule Name", + "cef.extensions.deviceCustomString5": "Business / Economy", + "cef.extensions.deviceCustomString5Label": "Matched Category", + "cef.extensions.deviceDirection": 0, + "cef.extensions.deviceReceiptTime": "2018-11-26T22:17:32.000Z", + "cef.extensions.ifname": "eth0", + "cef.extensions.inzone": "Internal", + "cef.extensions.layer_name": "Network", + "cef.extensions.layer_uuid": "b406b732-2437-4848-9741-6eae1f5bf112", + "cef.extensions.logid": "0", + "cef.extensions.loguid": "{0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001}", + "cef.extensions.match_id": "4", + "cef.extensions.nat_addtnl_rulenum": "1", + "cef.extensions.nat_rulenum": "4", + "cef.extensions.origin": "192.168.101.254", + "cef.extensions.originsicname": "CN=R80,O=R80_M..6u6bdo", + "cef.extensions.outzone": "External", + "cef.extensions.parent_rule": "0", + "cef.extensions.product": "VPN-1 & FireWall-1", + "cef.extensions.rule_action": "Accept", + "cef.extensions.rule_uid": "9e5e6e74-aa9a-4693-b9fe-53712dd27bea", + "cef.extensions.sequencenum": "1", + "cef.extensions.service_id": "https", + "cef.extensions.sourceAddress": "192.168.101.100", + "cef.extensions.sourcePort": 49363, + "cef.extensions.sourceTranslatedAddress": "192.168.103.254", + "cef.extensions.sourceTranslatedPort": 35398, + "cef.extensions.transportProtocol": "6", + "cef.extensions.version": "5", + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "destination.as.number": 8075, + "destination.as.organization.name": "Microsoft Corporation", + "destination.geo.city_name": "Des Moines", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 41.6006, + "destination.geo.location.lon": -93.6112, + "destination.geo.region_iso_code": "US-IA", + "destination.geo.region_name": "Iowa", + "destination.ip": "52.173.84.157", + "destination.nat.ip": "0.0.0.0", + "destination.nat.port": 0, + "destination.port": 443, + "event.action": "Accept", + "event.code": "Log", + "event.dataset": "cef.log", + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", + "fileset.name": "log", + "input.type": "log", + "log.offset": 0, + "message": "https", + "network.community_id": "1:yRLApDaheTmJZHL4UUDMjcHWAik=", + "network.direction": "inbound", + "network.transport": "6", + "observer.egress.zone": "External", + "observer.ingress.zone": "Internal", + "observer.interface.name": "eth0", + "observer.product": "VPN-1 & FireWall-1", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "rule.category": "Business / Economy", + "rule.uuid": "9e5e6e74-aa9a-4693-b9fe-53712dd27bea", + "service.type": "cef", + "source.ip": "192.168.101.100", + "source.nat.ip": "192.168.103.254", + "source.nat.port": 35398, + "source.port": 49363, + "tags": [ + "cef" + ] + }, + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.destinationPort": 25, + "cef.extensions.deviceAction": "Bypass", + "cef.extensions.deviceCustomDate2": "2020-04-11T10:42:13.000Z", + "cef.extensions.deviceCustomDate2Label": "Subscription expiration", + "cef.extensions.deviceCustomNumber1Label": "Email Recipients Number", + "cef.extensions.deviceCustomString1Label": "Email ID", + "cef.extensions.deviceCustomString4": "SMTP Policy Restrictions", + "cef.extensions.deviceCustomString4Label": "Email Control", + "cef.extensions.deviceCustomString5Label": "Email Session ID", + "cef.extensions.deviceDirection": 0, + "cef.extensions.deviceReceiptTime": "2018-12-19T09:22:10.000Z", + "cef.extensions.fileHash": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "cef.extensions.message": "Encrypted session", + "cef.extensions.sourcePort": 4001, + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "checkpoint.email_control": "SMTP Policy Restrictions", + "checkpoint.file_hash": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "checkpoint.subscription_expiration": "2020-04-11T10:42:13.000Z", + "destination.port": 25, + "event.action": "Bypass", + "event.code": "Log", + "event.dataset": "cef.log", + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", + "file.hash.sha1": "55f4a511e6f630a6b1319505414f114e7bcaf13d", + "fileset.name": "log", + "input.type": "log", + "log.offset": 875, + "message": "Encrypted session", + "network.direction": "inbound", + "observer.product": "VPN-1 & FireWall-1", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "service.type": "cef", + "source.port": 4001, + "tags": [ + "cef" + ] + }, + { + "cef.device.event_class_id": "Log", + "cef.device.product": "VPN-1 & FireWall-1", + "cef.device.vendor": "Check Point", + "cef.device.version": "Check Point", + "cef.extensions.baseEventCount": "12", + "cef.extensions.cp_app_risk": "High", + "cef.extensions.cp_severity": "Very-High", + "cef.extensions.deviceAction": "Drop", + "cef.extensions.deviceCustomIPv6Address2": "fd00::555", + "cef.extensions.deviceCustomIPv6Address2Label": "Source IPv6 Address", + "cef.extensions.deviceCustomIPv6Address3": "::1", + "cef.extensions.deviceCustomIPv6Address3Label": "Destination IPv6 Address", + "cef.extensions.deviceCustomNumber2": 5, + "cef.extensions.deviceCustomNumber2Label": "Duration in Seconds", + "cef.extensions.deviceFacility": "4", + "cef.extensions.fileHash": "580a783c1cb2b20613323f715d231a69", + "cef.name": "https", + "cef.severity": "Unknown", + "cef.version": "0", + "checkpoint.app_risk": "High", + "checkpoint.event_count": "12", + "checkpoint.file_hash": "580a783c1cb2b20613323f715d231a69", + "checkpoint.severity": "Very-High", + "destination.ip": "::1", + "event.action": "Drop", + "event.code": "Log", + "event.dataset": "cef.log", + "event.duration": 5000000000, + "event.module": "cef", + "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", + "event.risk_score": "4", + "event.severity": "4", + "file.hash.md5": "580a783c1cb2b20613323f715d231a69", + "fileset.name": "log", + "input.type": "log", + "log.offset": 1291, + "message": "https", + "observer.product": "VPN-1 & FireWall-1", + "observer.type": "Mobile", + "observer.vendor": "Check Point", + "observer.version": "Check Point", + "service.type": "cef", + "source.ip": "fd00::555", + "tags": [ + "cef" + ] + } +] \ No newline at end of file From e3f9f86e7ba84658615c6cdcbf7962b74ef36778 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 9 Mar 2020 18:21:09 +0100 Subject: [PATCH 04/11] Add docs --- filebeat/docs/modules/cef.asciidoc | 84 +++++++++++++++++++ .../filebeat/module/cef/_meta/docs.asciidoc | 84 +++++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/filebeat/docs/modules/cef.asciidoc b/filebeat/docs/modules/cef.asciidoc index d52feb78d91..c8e85e3e3c6 100644 --- a/filebeat/docs/modules/cef.asciidoc +++ b/filebeat/docs/modules/cef.asciidoc @@ -52,6 +52,90 @@ configuring the SMC. Testing was done with CEF logs from SMC version 6.6.1 and custom string mappings were taken from 'CEF Connector Configuration Guide' dated December 5, 2011. +[float] +==== Check Point devices + +This module will parse CEF data form Check Point devices as documented in +https://community.checkpoint.com/t5/Logging-and-Reporting/Log-Exporter-CEF-Field-Mappings/td-p/41060[Log Exporter CEF Field Mappings.] + +Check Point CEF extensions are mapped as follows: +[options="header"] +|======================================================================================================================= +| CEF Extension | CEF Label value | ECS Fields | Non-ECS Field | +| cp_app_risk | - | event.risk_score | checkpoint.app_risk | +| cp_severity | - | event.severity | checkpoint.severity | +| baseEventCount | - | - | checkpoint.event_count | +| deviceExternalId | - | observer.type | - | +| deviceFacility | - | observer.type | - | +| deviceInboundInterface | - | observer.ingress.interface.name | - | +| deviceOutboundInterface | - | observer.egress.interface.name | - | +| externalId | - | - | checkpoint.uuid | +| fileHash | - | file.hash.{md5,sha1} | checkpoint.file_hash | +| reason | - | - | checkpoint.termination_reason | +| checkrequestCookies | - | - | checkpoint.cookie | +| sourceNtDomain | - | dns.question.name | - | +| Signature | - | vulnerability.id | - | +| Recipient | - | destination.user.email | - | +| Sender | - | source.user.email | - | +| deviceCustomFloatingPoint1 | update version | observer.version | - | +| deviceCustomIPv6Address2 | source ipv6 address | source.ip | - | +| deviceCustomIPv6Address3 | destination ipv6 address | destination.ip | - | +.3+| deviceCustomNumber1 | elapsed time in seconds | host.uptime | - | + | email recipients number | - | checkpoint.email_recipients_num | + | payload | network.bytes | - | +.2+| deviceCustomNumber2 | icmp type | - | checkpoint.icmp_type | + | duration in seconds | event.duration | - | +| deviceCustomNumber3 | icmp code | - | checkpoint.icmp_code | +.6+| deviceCustomString1 | connectivity state | - | checkpoint.connectivity_state | + | application rule name | rule.name | - | + | threat prevention rule name | rule.name | - | + | voip log type | - | checkpoint.voip_log_type | + | dlp rule name | rule.name | - | + | email id | - | checkpoint.email_id | +.8+| deviceCustomString2 | category | - | checkpoint.category | + | email subject | - | checkpoint.email_subject | + | sensor mode | - | checkpoint.sensor_mode | + | protection id | - | checkpoint.protection_id | + | scan invoke type | - | checkpoint.integrity_av_invoke_type | + | update status | - | checkpoint.update_status | + | peer gateway | - | checkpoint.peer_gateway | + | categories | rule.category | - | +.4+| deviceCustomString6 | application name | process.name | - | + | virus name | - | checkpoint.virus_name | + | malware name | - | checkpoint.malware_name | + | malware family | - | checkpoint.malware_family | +.5+| deviceCustomString3 | user group | group.name | - | + | incident extension | - | checkpoint.incident_extension | + | protection type | - | checkpoint.protection_type | + | email spool id | - | checkpoint.email_spool_id | + | identity type | - | checkpoint.identity_type | +.9+| deviceCustomString4 | malware status | - | checkpoint.spyware_status | + | threat prevention rule id | rule.id | - | + | scan result | - | checkpoint.scan_result | + | tcp flags | - | checkpoint.tcp_flags | + | destination os | os.name | - | + | protection name | - | checkpoint.protection_name | + | email control | - | checkpoint.email_control | + | frequency | - | checkpoint.frequency | + | user response | - | checkpoint.user_status | +.4+| deviceCustomString5 | matched category | rule.category | - | + | vlan id | network.vlan.id | - | + | authentication method | - | checkpoint.auth_method | + | email session id | - | checkpoint.email_session_id | +| deviceCustomDate2 | subscription expiration | - | checkpoint.subscription_expiration | +| deviceFlexNumber1 | confidence | - | checkpoint.confidence_level | +.2+| deviceFlexNumber2 | performance impact | - | checkpoint.performance_impact | + | destination phone number | - | checkpoint.dst_phone_number | +| flexString1 | application signature id | - | checkpoint.app_sig_id | +.2+| flexString2 | malware action | event.action | - | + | attack information | event.action | - | +| rule_uid | - | rule.uuid | - | +| ifname | - | observer.interface.name | - | +| inzone | - | observer.ingress.zone | - | +| outzone | - | observer.egress.zone | - | +| product | - | observer.product | - | +|======================================================================================================================= + :has-dashboards!: diff --git a/x-pack/filebeat/module/cef/_meta/docs.asciidoc b/x-pack/filebeat/module/cef/_meta/docs.asciidoc index e962e86b735..2e9f06ed9c8 100644 --- a/x-pack/filebeat/module/cef/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cef/_meta/docs.asciidoc @@ -47,6 +47,90 @@ configuring the SMC. Testing was done with CEF logs from SMC version 6.6.1 and custom string mappings were taken from 'CEF Connector Configuration Guide' dated December 5, 2011. +[float] +==== Check Point devices + +This module will parse CEF data form Check Point devices as documented in +https://community.checkpoint.com/t5/Logging-and-Reporting/Log-Exporter-CEF-Field-Mappings/td-p/41060[Log Exporter CEF Field Mappings.] + +Check Point CEF extensions are mapped as follows: +[options="header"] +|======================================================================================================================= +| CEF Extension | CEF Label value | ECS Fields | Non-ECS Field | +| cp_app_risk | - | event.risk_score | checkpoint.app_risk | +| cp_severity | - | event.severity | checkpoint.severity | +| baseEventCount | - | - | checkpoint.event_count | +| deviceExternalId | - | observer.type | - | +| deviceFacility | - | observer.type | - | +| deviceInboundInterface | - | observer.ingress.interface.name | - | +| deviceOutboundInterface | - | observer.egress.interface.name | - | +| externalId | - | - | checkpoint.uuid | +| fileHash | - | file.hash.{md5,sha1} | checkpoint.file_hash | +| reason | - | - | checkpoint.termination_reason | +| checkrequestCookies | - | - | checkpoint.cookie | +| sourceNtDomain | - | dns.question.name | - | +| Signature | - | vulnerability.id | - | +| Recipient | - | destination.user.email | - | +| Sender | - | source.user.email | - | +| deviceCustomFloatingPoint1 | update version | observer.version | - | +| deviceCustomIPv6Address2 | source ipv6 address | source.ip | - | +| deviceCustomIPv6Address3 | destination ipv6 address | destination.ip | - | +.3+| deviceCustomNumber1 | elapsed time in seconds | host.uptime | - | + | email recipients number | - | checkpoint.email_recipients_num | + | payload | network.bytes | - | +.2+| deviceCustomNumber2 | icmp type | - | checkpoint.icmp_type | + | duration in seconds | event.duration | - | +| deviceCustomNumber3 | icmp code | - | checkpoint.icmp_code | +.6+| deviceCustomString1 | connectivity state | - | checkpoint.connectivity_state | + | application rule name | rule.name | - | + | threat prevention rule name | rule.name | - | + | voip log type | - | checkpoint.voip_log_type | + | dlp rule name | rule.name | - | + | email id | - | checkpoint.email_id | +.8+| deviceCustomString2 | category | - | checkpoint.category | + | email subject | - | checkpoint.email_subject | + | sensor mode | - | checkpoint.sensor_mode | + | protection id | - | checkpoint.protection_id | + | scan invoke type | - | checkpoint.integrity_av_invoke_type | + | update status | - | checkpoint.update_status | + | peer gateway | - | checkpoint.peer_gateway | + | categories | rule.category | - | +.4+| deviceCustomString6 | application name | process.name | - | + | virus name | - | checkpoint.virus_name | + | malware name | - | checkpoint.malware_name | + | malware family | - | checkpoint.malware_family | +.5+| deviceCustomString3 | user group | group.name | - | + | incident extension | - | checkpoint.incident_extension | + | protection type | - | checkpoint.protection_type | + | email spool id | - | checkpoint.email_spool_id | + | identity type | - | checkpoint.identity_type | +.9+| deviceCustomString4 | malware status | - | checkpoint.spyware_status | + | threat prevention rule id | rule.id | - | + | scan result | - | checkpoint.scan_result | + | tcp flags | - | checkpoint.tcp_flags | + | destination os | os.name | - | + | protection name | - | checkpoint.protection_name | + | email control | - | checkpoint.email_control | + | frequency | - | checkpoint.frequency | + | user response | - | checkpoint.user_status | +.4+| deviceCustomString5 | matched category | rule.category | - | + | vlan id | network.vlan.id | - | + | authentication method | - | checkpoint.auth_method | + | email session id | - | checkpoint.email_session_id | +| deviceCustomDate2 | subscription expiration | - | checkpoint.subscription_expiration | +| deviceFlexNumber1 | confidence | - | checkpoint.confidence_level | +.2+| deviceFlexNumber2 | performance impact | - | checkpoint.performance_impact | + | destination phone number | - | checkpoint.dst_phone_number | +| flexString1 | application signature id | - | checkpoint.app_sig_id | +.2+| flexString2 | malware action | event.action | - | + | attack information | event.action | - | +| rule_uid | - | rule.uuid | - | +| ifname | - | observer.interface.name | - | +| inzone | - | observer.ingress.zone | - | +| outzone | - | observer.egress.zone | - | +| product | - | observer.product | - | +|======================================================================================================================= + :has-dashboards!: From 92a30c0ba85a078466fc6daf5a94471aaeca2fa3 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 9 Mar 2020 18:34:01 +0100 Subject: [PATCH 05/11] Changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index cafe44d8793..98dcc5c45d0 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -186,6 +186,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Release ActiveMQ module as GA. {issue}17047[17047] {pull}17049[17049] - Improve ECS categorization field mappings in iptables module. {issue}16166[16166] {pull}16637[16637] - Add Filebeat Okta module. {pull}16362[16362] +- Add custom string mapping to CEF module to support Check Point devices. {issue}16041[16041] {pull}NNNN[NNNN] *Heartbeat* From aae607156ff1115ecc54649ac7bc8eca222bfbda Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 9 Mar 2020 18:44:16 +0100 Subject: [PATCH 06/11] Add PR number --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 98dcc5c45d0..45baf01c723 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -187,6 +187,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS categorization field mappings in iptables module. {issue}16166[16166] {pull}16637[16637] - Add Filebeat Okta module. {pull}16362[16362] - Add custom string mapping to CEF module to support Check Point devices. {issue}16041[16041] {pull}NNNN[NNNN] +- Add custom string mapping to CEF module to support Check Point devices. {issue}16041[16041] {pull}16907[16907] *Heartbeat* From 2db8b7aba42b3da8e8389c8e12d067ce376a3b40 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Tue, 10 Mar 2020 16:37:34 +0100 Subject: [PATCH 07/11] Update decode_cef golden files --- .../decode_cef/testdata/samples.log.golden.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json index c5ca7c5a93f..3f1a1c38398 100644 --- a/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json +++ b/x-pack/filebeat/processors/decode_cef/testdata/samples.log.golden.json @@ -89,7 +89,6 @@ "version": "1.2.3" }, "extensions": { - "DeviceCustomNumber2": 205, "TrendMicroDsDetectionConfidence": "95", "TrendMicroDsFileMD5": "44D88612FEA8A8F36DE82E1278ABB02F", "TrendMicroDsFileSHA1": "3395856CE81F2B7382DEE72602F798B642F14140", @@ -100,6 +99,7 @@ "deviceAction": "Delete", "deviceCustomNumber1": 1, "deviceCustomNumber1Label": "Host ID", + "deviceCustomNumber2": 205, "deviceCustomNumber2Label": "Quarantine File Size", "deviceCustomString6": "ContainerImageName | ContainerName | ContainerID", "deviceCustomString6Label": "Container", @@ -450,9 +450,9 @@ "version": "NS10.0" }, "extensions": { - "DeviceCustomNumber2": 205, "deviceAction": "blocked", "deviceCustomNumber1": 233, + "deviceCustomNumber2": 205, "deviceCustomString1": "profile1", "deviceCustomString2": "PPE0", "deviceCustomString3": "AjSZM26h2M+xL809pON6C8joebUA000", @@ -497,9 +497,9 @@ "version": "NS10.0" }, "extensions": { - "DeviceCustomNumber2": 535, "deviceAction": "not blocked", "deviceCustomNumber1": 465, + "deviceCustomNumber2": 535, "deviceCustomString1": "profile1", "deviceCustomString2": "PPE0", "deviceCustomString3": "IliG4Dxp1SjOhKVRDVBXmqvAaIcA000", @@ -544,9 +544,9 @@ "version": "NS10.0" }, "extensions": { - "DeviceCustomNumber2": 610, "deviceAction": "transformed", "deviceCustomNumber1": 652, + "deviceCustomNumber2": 610, "deviceCustomString1": "pr_ffc", "deviceCustomString2": "PPE0", "deviceCustomString3": "li8MdGfW49uG8tGdSV85ech41a0A000", @@ -591,9 +591,9 @@ "version": "NS10.0" }, "extensions": { - "DeviceCustomNumber2": 610, "deviceAction": "transformed", "deviceCustomNumber1": 653, + "deviceCustomNumber2": 610, "deviceCustomString1": "pr_ffc", "deviceCustomString2": "PPE0", "deviceCustomString3": "li8MdGfW49uG8tGdSV85ech41a0A000", @@ -638,9 +638,9 @@ "version": "NS10.0" }, "extensions": { - "DeviceCustomNumber2": 205, "deviceAction": "not blocked", "deviceCustomNumber1": 224, + "deviceCustomNumber2": 205, "deviceCustomString1": "pr_ffc", "deviceCustomString2": "PPE0", "deviceCustomString3": "POousP7CIMW5nwZ5Rs4nq5DND0sA000", From ab3418c1fea8d828d43fffac42d4e0acf0dba243 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Thu, 12 Mar 2020 00:07:40 +0100 Subject: [PATCH 08/11] Remove temporary ECS fields after rebase --- filebeat/docs/fields.asciidoc | 15 --------------- x-pack/filebeat/module/cef/fields.go | 2 +- x-pack/filebeat/module/cef/log/_meta/fields.yml | 5 ----- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e37e30f96eb..d83c81842d1 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -5293,21 +5293,6 @@ type: keyword -- -*`observer.ingress.zone`*:: -+ --- --- - -*`observer.egress.zone`*:: -+ --- --- - -*`observer.interface.name`*:: -+ --- --- - [[exported-fields-cisco]] == Cisco fields diff --git a/x-pack/filebeat/module/cef/fields.go b/x-pack/filebeat/module/cef/fields.go index f4d5d5f2d46..217d805818d 100644 --- a/x-pack/filebeat/module/cef/fields.go +++ b/x-pack/filebeat/module/cef/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCef returns asset data. // This is the base64 encoded gzipped contents of module/cef. func AssetCef() string { - return "eJy8Wc9v6zYMvvevILDLdmiAd3iXHAY8pO0WDN0L0O5dBVWmHS62qEm02+yvHyTnV9u0Vu1iOQWR9H0fKVIilUvY4HYOBsvLhou2xgsAIalxDovrmwuAAoPx5ITYzuHXCwCA2zQRSvbg0SB1ZKs4G2quAnCHHu62oeZqBvdrhB4XdFEE6NAW7BNKcGioJAMlYV0EIBunUCQCYZA17kfi1wINF6gMluA8GwxhB+M8d1RgmF3Abv48/R4/l2B1g/Oo1KBjsnIYApCtwzlUnlt38muBpW5rUQlqDqWuAz4bfuWN/eemFxu9cnPgg0UbhBu4Ex+d1GjnyFbhZOFLzae6O/JtUFQ8G9wr3+D2kf3LsXcExs+PiAjLq8PA6y97crNGs/kfnbaIfLBKXjO918Jzr80y3aadU57CZqzbvjlXk9EpEiPO7LWTXtIF7NCTbD+DUtYetcAeMoedqglREnM0UGW1tB5heQWPazLrlHX6RNajDlCgoBEs4GFQVitr1aCsebSulQ4hzklYaGUvxHkWNlxDG7AYkGG0YMV+9L4sduuHaNiWVKA1qGrssB5Nd8CBhJMc7huyw4aytWiEOpKtCqIFJ2g4IEFCGqTmDY2mW/5xvUMYoCmCKLdmi8q2zQP6sYRXGIRsH0rL1eUqQg5QY6OpVoateB69t9e2IosJMYtufEIvraC3uu6BYHmVxefRkCO0EqKDz3LXbKv3iP9MGwNcwhEsiztgCMT282zeAebaHhzzJ3o8wWVztw9/o5HRYdUz9iBDhB1aUYZbe54uf38TUgAdAhvS8UZ4JOnvjFj3vS+jpBrVWof1WJtvqEaIAPDz3e/fvgB7uL36+ssQrcd/WrRm9GVwZ7Q9ogywkWmcitCjPL1c3K7S1ByWWBmPZ4mrh1iKeP3K9m17cnJkB5JlljWJU+GToI2ZPDpU2DdaYsiyp4pijhZa9CC/YBWLL6U7RbbjDU6yPUVOj5NjvkP0qtKCj/p8sJJ7j+1WU7zaotExISMa3KFpo0HwWw87qMCX0XOxoqHG6fHn08pzrBlT4XYEhR50SMVh7YSz+URAo+tHnWrcfOL4w+ht37e4x/I6wu03RotoM9RdnEiZEoH3W5d4j3CpfI6ddl/U5ysKRlvlMbT16KBI+dBDDJGhDexV89YRl0OWIKAZPuemdnH3H+rcdtGYqvU2jOW83cV0jzJkYPtwWKrwyZFPdfBZ7uJ1D/GqbTxCpOn7wD6lGVAkxqmy1tVo++8XK3DabFAg4QzxpXYqSVYedRh/s9wfkaBHGqBuXfTRxO3+K4Hk7XYb0E+lC+hjojq2YSh72nb8IX39tCugB4/m/k1syqncv4Fl9GH7BJ1Ctk/PD9CVuqF69Bm0J+xRhpzJ5FTN1aSr5QcvV7HqTyvOxeThoQDL2aGgOw3Jz35VvH4Sr3dvzZeHd+Yj9XsPiWf9ZJz60KPiWyAfumDOV6ZldjS+Udn+y3bC+lpv0avsVH8HY5odNVeTJHA1yYRGi1nnFqVnEawWpYtCbK18W+NbTy7ZWJNB+v5o6vpAZtrGcivTItRpHztH3/+XNRbEc9FmNj1nASK90ubN6iofZFKchv6xYlpkBPQdxUZwgpAOfXYbf/jyE9x/v/p+KHqE4XpxB19mX0HbAjw23J1cqocIeoiC0c/IVh5DmD2Lp1ezMGcSWUFfaoOzFN3/BQAA//9c6pxr" + return "eJy8mctu7DYPx/d5Ci6/b5EAXXQziwIHubSDIsUASc9WUGTaw44sqhLtZPr0heS5nWQSOXbQWQW29fuTFCWRyiVscLsAg/Vly1Vn8QJASCwu4Pr27gKgwmgCeSF2C/jlAgDgPn8INQcIaJB6ck36Giw3EbjHAA/baLm5gsc1wsAFXVURenQVh0yJHg3VZKAmtFUEcukTSkIgDLLG/Zv0Z4WGK1QGa/CBDca4w/jAPVUYry5g9/0iP0+/S3C6xUWy1KBncnJ4BSBbjwtoAnf+5GmFte6sqIxaQK1txB9ev4nG/nc3GJuicnfQg+suCrfwICEFqdXek2viycDXNp/a3VPooqLqh5d7yze4febw+t0HBqbf90SE5c3hxds/9uJmjWbzHwbtOunBKkfNDFGLP0btamTYtPcqUNxMDds37y0ZnTMxca7eBum1XMQeA8n2KyRlHVAL7JFj1KmZkSVpjUZqnJYuICxv4HlNZp1XnT4x61lHqFDQCFbwVDSrk7VqUdY82a6VjjF9k1noZG+IDyxs2EIXsSqYYbRgw2HyvFzvxpdk2NVUoTOoLPZoJ8sdOJA5OeChJVd2lJ1DI9STbFUULTjDhgMJMqkozRuaLLf8/XZHKMhUUZRfs0PluvYJw1TBG4xCbkil5epylZAFaWw1WWXYSeDJc3vrGnKYiaPkpi/opRMMTtsBBMubUXoBDXlCJzEF+Ky2Zdd8JPxHnhjgGo6wUdoRYyR2X+fzDjjW9+iZvzDiGTdau3v6C41MTqtBcYCUBHt0ogx37rzc+PnNpAg6Rjak04nwTDKcGanu+9iMmiyqtY7rqT7fkUVIAPjfw2/ffgIOcH/z8/9LsgH/7tCZyYfBg9HuSCmokWm9SuhJkV5e36/yp2NUUmU8XSWNLqlU6fiV7fv+jFkjO8got5zJmgpfBF1ayZNThUOrJaUsB2oordFKiy7qCzap+FK6V+R63uAs33PmDJwx7nvEoBot+KzPJyv5j9TuNaWjLTmdFmSiwQOaLjkEvw7YogWhTpFLFQ21Xk/fn1aBU82YC7cjFAZoyYrD2Bl784kBrbbPOte444XTg8nTvm9xj+V1wu0nRotoU+ouTkyZk4GPW591j7hcPqdOeyjqx1sUjXYqYOzs5KTI62FAlMTQRQ6qfW+LGyOWEdCW97m5Xdzjpzq3XTbmar2LUzXvdzk9UEoOdk+HoQpfPIVcB5/Vrt72EG/axiMif75P7FOZgkVivKqtbib7/3i9Aq/NBgUyp6SX26lssgqo4/ST5fFIgoFUkO58itHM6f4zQ8bNdhcxzJWLGNJC9exiafV03fRN+vZlV0AXt+bhTmzOrjzcgY3ow/YLdI7Yfnl+Qq7WLdnJe9BecKCUgsnkleVm1tHynZerVPXnEedy8nBRgPXVoaA7TcmvvlW8fZGgd3fNl4d75qP0RxeJZ+NkvPrUpeJ7kE8dMOcr03p0Nr5T2f7DbsZ4q7cY1Oil/gFjnh+Wm1kmcDPLhVaLWY8tSs8SnBalq0qcVaGz+N6Vy2jWbMjQH80dH8nMm1juZF6Geh1S5xiG/2VNhQSuupFNz1lAklfavFtdjYfMytM4XFbMy4yIoafUCM4wpMdQbOP/DQAA//8I9Guw" } diff --git a/x-pack/filebeat/module/cef/log/_meta/fields.yml b/x-pack/filebeat/module/cef/log/_meta/fields.yml index 16e54d71cf1..40f6cdb4bfb 100644 --- a/x-pack/filebeat/module/cef/log/_meta/fields.yml +++ b/x-pack/filebeat/module/cef/log/_meta/fields.yml @@ -253,8 +253,3 @@ - name: version type: keyword - -# TODO: Update to ECS 1.5 and remove. -- name: observer.ingress.zone -- name: observer.egress.zone -- name: observer.interface.name From 98f9f17fd424d8631cce139c93e9d085b722c194 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 13 Mar 2020 15:12:15 +0100 Subject: [PATCH 09/11] observer.interface -> observer.ingress.interface --- filebeat/docs/modules/cef.asciidoc | 2 +- x-pack/filebeat/module/cef/_meta/docs.asciidoc | 2 +- x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml | 2 +- .../filebeat/module/cef/log/test/checkpoint.log-expected.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/filebeat/docs/modules/cef.asciidoc b/filebeat/docs/modules/cef.asciidoc index c8e85e3e3c6..bb5b77dee42 100644 --- a/filebeat/docs/modules/cef.asciidoc +++ b/filebeat/docs/modules/cef.asciidoc @@ -130,7 +130,7 @@ Check Point CEF extensions are mapped as follows: .2+| flexString2 | malware action | event.action | - | | attack information | event.action | - | | rule_uid | - | rule.uuid | - | -| ifname | - | observer.interface.name | - | +| ifname | - | observer.ingress.interface.name | - | | inzone | - | observer.ingress.zone | - | | outzone | - | observer.egress.zone | - | | product | - | observer.product | - | diff --git a/x-pack/filebeat/module/cef/_meta/docs.asciidoc b/x-pack/filebeat/module/cef/_meta/docs.asciidoc index 2e9f06ed9c8..d3f97e011dd 100644 --- a/x-pack/filebeat/module/cef/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cef/_meta/docs.asciidoc @@ -125,7 +125,7 @@ Check Point CEF extensions are mapped as follows: .2+| flexString2 | malware action | event.action | - | | attack information | event.action | - | | rule_uid | - | rule.uuid | - | -| ifname | - | observer.interface.name | - | +| ifname | - | observer.ingress.interface.name | - | | inzone | - | observer.ingress.zone | - | | outzone | - | observer.egress.zone | - | | product | - | observer.product | - | diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml index 1cf4399dbb5..b99f164f090 100644 --- a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml @@ -232,7 +232,7 @@ processors: to: rule.uuid - name: ifname - to: observer.interface.name + to: observer.ingress.interface.name - name: inzone to: observer.ingress.zone diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json index e9753166af1..a6e346e2809 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json @@ -69,8 +69,8 @@ "network.direction": "inbound", "network.transport": "6", "observer.egress.zone": "External", + "observer.ingress.interface.name": "eth0", "observer.ingress.zone": "Internal", - "observer.interface.name": "eth0", "observer.product": "VPN-1 & FireWall-1", "observer.vendor": "Check Point", "observer.version": "Check Point", From d17acce2cf476d262ed1a526496105873bd3c8d2 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Fri, 13 Mar 2020 17:24:16 +0100 Subject: [PATCH 10/11] Try to populate some event categorisation fields --- .../module/cef/log/ingest/cp-pipeline.yml | 29 +++++++++++++++++-- .../cef/log/test/checkpoint.log-expected.json | 5 ++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml index b99f164f090..f3f38355ed9 100644 --- a/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml +++ b/x-pack/filebeat/module/cef/log/ingest/cp-pipeline.yml @@ -297,10 +297,35 @@ processors: # checkpoint.file_hash can be either MD5 or SHA1. - set: - field: 'file.hash.md5' + field: file.hash.md5 value: '{{checkpoint.file_hash}}' if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==32' - set: - field: 'file.hash.sha1' + field: file.hash.sha1 value: '{{checkpoint.file_hash}}' if: 'ctx.checkpoint?.file_hash != null && ctx.checkpoint.file_hash.length()==40' + + # Event kind is 'event' by default. 'alert' when a risk score and rule info + # is present. + - set: + field: event.kind + value: event + - set: + field: event.kind + value: alert + if: 'ctx.cef?.extensions?.cp_app_risk != null && ctx.rule != null' + + # Set event.category to network/malware/intrusion_detection depending on which + # fields have been populated. + - set: + field: event.category + value: network + if: 'ctx.source?.ip != null && ctx.destination?.ip != null' + - set: + field: event.category + value: malware + if: 'ctx.checkpoint?.protection_id != null || ctx.checkpoint?.malware_name != null || ctx.checkpoint?.malware_family != null || ctx.checkpoint?.spyware_status != null' + - set: + field: event.category + value: intrusion_detection + if: 'ctx.event?.category != "malware" && (ctx.checkpoint?.protection_type != null || ctx.cef.extensions?.flexString2Label == "Attack Information")' diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json index a6e346e2809..6a33734f9ce 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json @@ -57,8 +57,10 @@ "destination.nat.port": 0, "destination.port": 443, "event.action": "Accept", + "event.category": "network", "event.code": "Log", "event.dataset": "cef.log", + "event.kind": "event", "event.module": "cef", "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Accept destinationTranslatedAddress=0.0.0.0 destinationTranslatedPort=0 deviceDirection=0 rt=1543270652000 sourceTranslatedAddress=192.168.103.254 sourceTranslatedPort=35398 spt=49363 dpt=443 cs2Label=Rule Name layer_name=Network layer_uuid=b406b732-2437-4848-9741-6eae1f5bf112 match_id=4 parent_rule=0 rule_action=Accept rule_uid=9e5e6e74-aa9a-4693-b9fe-53712dd27bea ifname=eth0 logid=0 loguid={0x5bfc70fc,0x1,0xfe65a8c0,0xc0000001} origin=192.168.101.254 originsicname=CN\\=R80,O\\=R80_M..6u6bdo sequencenum=1 version=5 dst=52.173.84.157 inzone=Internal nat_addtnl_rulenum=1 nat_rulenum=4 outzone=External product=VPN-1 & FireWall-1 proto=6 service_id=https src=192.168.101.100 cs5Label=Matched Category cs5=Business / Economy deviceCustomDate2=1508150533713 deviceCustomDate2Label=This field is made up", "fileset.name": "log", @@ -114,6 +116,7 @@ "event.action": "Bypass", "event.code": "Log", "event.dataset": "cef.log", + "event.kind": "event", "event.module": "cef", "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Bypass cn1Label=Email Recipients Number cs1Label=Email ID cs4Label=Email Control cs4=SMTP Policy Restrictions cs5Label=Email Session ID deviceDirection=0 msg=Encrypted session rt=1545211330000 spt=4001 dpt=25 fileHash=55f4a511e6f630a6b1319505414f114e7bcaf13d deviceCustomDate2=Apr 11 2020 10:42:13 deviceCustomDate2Label=Subscription expiration", "file.hash.sha1": "55f4a511e6f630a6b1319505414f114e7bcaf13d", @@ -157,9 +160,11 @@ "checkpoint.severity": "Very-High", "destination.ip": "::1", "event.action": "Drop", + "event.category": "network", "event.code": "Log", "event.dataset": "cef.log", "event.duration": 5000000000, + "event.kind": "event", "event.module": "cef", "event.original": "CEF:0|Check Point|VPN-1 & FireWall-1|Check Point|Log|https|Unknown|act=Drop cp_app_risk=High cp_severity=Very-High baseEventCount=12 deviceFacility=4 c6a2=fd00::555 c6a2Label=Source IPv6 Address c6a3=::1 c6a3Label=Destination IPv6 Address fileHash=580a783c1cb2b20613323f715d231a69 cn2=5 cn2Label=Duration in Seconds", "event.risk_score": "4", From 81a770255d1976acb3c577b816530170546815d7 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Wed, 18 Mar 2020 11:06:49 +0100 Subject: [PATCH 11/11] Update golden files --- .../module/cef/log/test/checkpoint.log-expected.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json index 6a33734f9ce..0cc100922d0 100644 --- a/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json +++ b/x-pack/filebeat/module/cef/log/test/checkpoint.log-expected.json @@ -76,6 +76,12 @@ "observer.product": "VPN-1 & FireWall-1", "observer.vendor": "Check Point", "observer.version": "Check Point", + "related.ip": [ + "52.173.84.157", + "0.0.0.0", + "192.168.101.100", + "192.168.103.254" + ], "rule.category": "Business / Economy", "rule.uuid": "9e5e6e74-aa9a-4693-b9fe-53712dd27bea", "service.type": "cef", @@ -128,6 +134,9 @@ "observer.product": "VPN-1 & FireWall-1", "observer.vendor": "Check Point", "observer.version": "Check Point", + "related.hash": [ + "55f4a511e6f630a6b1319505414f114e7bcaf13d" + ], "service.type": "cef", "source.port": 4001, "tags": [ @@ -178,6 +187,9 @@ "observer.type": "Mobile", "observer.vendor": "Check Point", "observer.version": "Check Point", + "related.hash": [ + "580a783c1cb2b20613323f715d231a69" + ], "service.type": "cef", "source.ip": "fd00::555", "tags": [