[7.x](backport #25674) [Filebeat] Fix bad append for abusechmalware (#…

…25853) * [Filebeat] Fix bad append for abusechmalware (#25674) * 25151: Fix bad append for abusechmalware * update changelog (cherry picked from commit fafe6fb) * Update CHANGELOG.next.asciidoc Co-authored-by: Alex Resnick <adr8292@gmail.com> Co-authored-by: Marius Iversen <marius.iversen@elastic.co>
elastic · May 25, 2021 · 5e60010 · 5e60010
1 parent f5e853f
commit 5e60010
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 5 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -276,6 +276,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Nginx module pipelines. {issue}19088[19088] {pull}24699[24699]
 - Remove space from field `sophos.xg.trans_src_ ip`. {issue}25154[25154] {pull}25250[25250]
 - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
+- Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 
 *Heartbeat*
 

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/abusemalware/ingest/pipeline.yml
@@ -93,8 +93,8 @@ processors:
     if: ctx?.threatintel?.indicator?.file?.pe?.imphash != null
 - append:
     field: related.hash
-    value: '{{ threatintel.indicator.file.pe.tlsh }}'
-    if: ctx?.threatintel?.indicator?.file?.pe?.tlsh != null
+    value: '{{ threatintel.indicator.file.hash.tlsh }}'
+    if: ctx?.threatintel?.indicator?.file?.hash?.tlsh != null
 
 ######################
 # Cleanup processors #

diff --git a/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/abusemalware/test/abusechmalware.ndjson.log-expected.json
@@ -10,6 +10,7 @@
         "input.type": "log",
         "log.offset": 0,
         "related.hash": [
+            "1344D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "48a6aee18bcfe9058b35b1018832aef1c9efd8f50ac822f49abb484a5e2a4b1f",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG5:X5DpBw/KViMTB1MnEWk0115JW",
             "68aea345b134d576ccdef7f06db86088",
@@ -41,6 +42,7 @@
         "input.type": "log",
         "log.offset": 580,
         "related.hash": [
+            "4E44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGY:X5DpBw/KViMTB1MnEWk0115Jr",
             "68aea345b134d576ccdef7f06db86088",
             "7b4c77dc293347b467fb860e34515163",
@@ -75,6 +77,7 @@
             "373d34874d7bc89fd4cefa6272ee80bf",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGG:X5DpBw/KViMTB1MnEWk0115Jd",
             "68aea345b134d576ccdef7f06db86088",
+            "7544D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "b0e914d1bbe19433cc9df64ea1ca07fe77f7b150b511b786e46e007941a62bd7"
         ],
         "service.type": "threatintel",
@@ -106,6 +109,7 @@
         "input.type": "log",
         "log.offset": 1904,
         "related.hash": [
+            "5554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717",
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ9:0h3eZgRQCcw+MN54dEq7kqRtoLZH",
             "68aea345b134d576ccdef7f06db86088",
             "7483e834a73fb6817769596fe4c0fa01d28639f52bbbdc2b8a56c36d466dd7f8",
@@ -137,6 +141,7 @@
         "input.type": "log",
         "log.offset": 2493,
         "related.hash": [
+            "3CE0C002AB26C036500D154C221655B3B871911503CA14E6A6824BEA765D4A3290D190",
             "3e988e32b0c3c230d534e286665b89a5",
             "6:TE6ll8uXi0jIAv6BHvPuA7RKTmOQamsQMGvMQgTYbtsWsQ72hCqPZG/:TTll8uTo5uA7RKtQamsS0QJfsQ7mCR",
             "760e729426fb115b967a41e5a6f2f42d7a52a5cee74ed99065a6dc39bf89f59b"
@@ -166,6 +171,7 @@
         "input.type": "log",
         "log.offset": 3054,
         "related.hash": [
+            "0D44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGI:X5DpBw/KViMTB1MnEWk0115JH",
             "68aea345b134d576ccdef7f06db86088",
             "86655c0bcf9b21b5efc682f58eb80f42811042ba152358e1bfbbb867315a60ac",
@@ -200,6 +206,7 @@
         "input.type": "log",
         "log.offset": 3798,
         "related.hash": [
+            "2554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717",
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJ1:0h3eZgRQCcw+MN54dEq7kqRtoLZL",
             "68aea345b134d576ccdef7f06db86088",
             "e91c9e11d3ce4f55fabd7196279367482d2fabfa32df81e614b15fc53b4e26be",
@@ -234,6 +241,7 @@
             "44325fd5bdda2e2cdea07c3a39953bb1",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Jg",
             "68aea345b134d576ccdef7f06db86088",
+            "A044D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "beedbbcacfc34b5edd8c68e3e4acf364992ebbcd989548e09e38fa03c5659bac"
         ],
         "service.type": "threatintel",
@@ -262,6 +270,7 @@
         "input.type": "log",
         "log.offset": 4967,
         "related.hash": [
+            "4544D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "4c549051950522a3f1b0814aa9b1f6d1",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG4:X5DpBw/KViMTB1MnEWk0115Jv",
             "68aea345b134d576ccdef7f06db86088",
@@ -297,6 +306,7 @@
             "426be5e085e6bbad8430223dc89d8d3ced497133f8d478fd00005bcbb73399d4",
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJw:0h3eZgRQCcw+MN54dEq7kqRtoLZW",
             "68aea345b134d576ccdef7f06db86088",
+            "9454CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717",
             "d7333113098d88b6a5dd5b8eb24f9b87"
         ],
         "service.type": "threatintel",
@@ -328,6 +338,7 @@
             "25093afdaeb3ea000743ab843360a6b64f58c0a1ab950072ba6528056735deb9",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGe:X5DpBw/KViMTB1MnEWk0115JR",
             "68aea345b134d576ccdef7f06db86088",
+            "F344D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "c8dbb261c1f450534c3693da2f4b479f"
         ],
         "service.type": "threatintel",
@@ -359,6 +370,7 @@
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115J7",
             "68aea345b134d576ccdef7f06db86088",
             "714953f1d0031a4bb2f0c44afd015931",
+            "F644D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "b3327a96280365e441057f490df6261c9a2400fd63719eb9a7a0c9db95beecc5"
         ],
         "service.type": "threatintel",
@@ -390,6 +402,7 @@
             "20fd22742500d4cec123398afc3d3672",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGc:X5DpBw/KViMTB1MnEWk0115JP",
             "68aea345b134d576ccdef7f06db86088",
+            "BE44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "e92b54904391c171238863b584355197ba4508f73320a8e89afbb5425fc2dc4b"
         ],
         "service.type": "threatintel",
@@ -420,6 +433,7 @@
         "related.hash": [
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGf:X5DpBw/KViMTB1MnEWk0115Jo",
             "68aea345b134d576ccdef7f06db86088",
+            "CC44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "aa81ceea053797a6f8c38a0f2f9b80b0",
             "dd15e74b3cd3a4fdb5f47adefd6f90e27d5a20e01316cc791711f6dce7c0f52e"
         ],
@@ -452,6 +466,7 @@
             "0fae1eeabc4f5e07bd16f7851aec5ab6032d407c7ff0270f2b6e85c2a3efebd1",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGD:X5DpBw/KViMTB1MnEWk0115JY",
             "68aea345b134d576ccdef7f06db86088",
+            "8C44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "a2ce6795664c0fa93b07fa54ba868991"
         ],
         "service.type": "threatintel",
@@ -484,6 +499,7 @@
             "07a9d84c0b2c8cf1fd90ab409b9399d06920ab4b6efb647b5a3b9bef1045ee7e",
             "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKk:W5MT4WNaHy9P1FjbrjlKk",
             "68aea345b134d576ccdef7f06db86088",
+            "6B54CF217A53C826F5E800FCA6E9878914167F346F44A4C773D40F6AA8759E2EF2B317",
             "9b9bac158dacb9c2f5511e9c464a7de4"
         ],
         "service.type": "threatintel",
@@ -513,6 +529,7 @@
         "log.offset": 9611,
         "related.hash": [
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGo:X5DpBw/KViMTB1MnEWk0115Jj",
+            "6644D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "68aea345b134d576ccdef7f06db86088",
             "708c0193aec6354af6877f314d4b0e3864552bac77258bee9ee5bf886a116df5",
             "e48e3fa5e0f7b21c1ecf1efc81ff91e8"
@@ -543,6 +560,7 @@
         "input.type": "log",
         "log.offset": 10191,
         "related.hash": [
+            "0754CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717",
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJy:0h3eZgRQCcw+MN54dEq7kqRtoLZM",
             "68aea345b134d576ccdef7f06db86088",
             "8957f5347633ab4b10c2ae4fb92c8572",
@@ -578,7 +596,8 @@
             "09cc76b7077b4d5704e46e864575ff03",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG/:X5DpBw/KViMTB1MnEWk0115Js",
             "68aea345b134d576ccdef7f06db86088",
-            "94ca186561b13fa9b1bf15f7e66118debc686b40d2a62a5cf4b3c6ca6ee1c7a1"
+            "94ca186561b13fa9b1bf15f7e66118debc686b40d2a62a5cf4b3c6ca6ee1c7a1",
+            "BB44D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717"
         ],
         "service.type": "threatintel",
         "tags": [
@@ -609,7 +628,8 @@
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJQ:0h3eZgRQCcw+MN54dEq7kqRtoLZ+",
             "68aea345b134d576ccdef7f06db86088",
             "909f890dbc5748845cf06d0fb0b73a5c0cb17761f37e9cd4810eea0d0eb8627f",
-            "98a1cdf7de4232363f1d1e0f33dbfd99"
+            "98a1cdf7de4232363f1d1e0f33dbfd99",
+            "C554CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717"
         ],
         "service.type": "threatintel",
         "tags": [
@@ -637,6 +657,7 @@
         "input.type": "log",
         "log.offset": 11952,
         "related.hash": [
+            "1654CF22E642C926F1E900FCB2A98B4451257E355F40F4D777C40FABA835AE2AF27717",
             "6144:0hlBeZgR9LqvgFcwNAwhGV52n5Dv4JdEqvQykqRqYdBx8pRA7OZJh:0h3eZgRQCcw+MN54dEq7kqRtoLZ/",
             "68aea345b134d576ccdef7f06db86088",
             "8a51830c1662513ba6bd44e2f7849547",
@@ -670,6 +691,7 @@
         "log.offset": 12544,
         "related.hash": [
             "3b9698b6c18bcba15ee33378440dd3f42509730e6b1d2d5832c71a74b1920e51",
+            "5454CF217A53C826F5E800FCA6E9878925167F346F44A4C373D40F6AA8759E2DF2B317",
             "6144:WlLMUG2gFWLDFO9vNa11y3NPcJufFFTXNZrjJTKS:W5MT4WNaHy9P1FjbrjlKS",
             "68aea345b134d576ccdef7f06db86088",
             "ae21d742a8118d6b86674aa5370bd6a7"
@@ -700,6 +722,7 @@
         "input.type": "log",
         "log.offset": 13113,
         "related.hash": [
+            "6044D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JG8:X5DpBw/KViMTB1MnEWk0115Jr",
             "68aea345b134d576ccdef7f06db86088",
             "78c9d88d24ed1d982a83216eed1590f6",
@@ -733,6 +756,7 @@
         "related.hash": [
             "236577d5d83e2a8d08623a7a7f724188",
             "6144:X1G3WVIOY6Bdjehj+qudd96ou/6mv5wdC:X1GmSafShjYdd96z/6cwdC",
+            "8D34BE41B28B8B4BD163163C2976D1F8953CFC909761CE693B64B22F0F739D0892E7A5",
             "8cd28fed7ebdcd79ea2509dca84f0a727ca28d4eaaed5a92cd10b1279ff16afa",
             "ed2860c18f5483e3b5388bad75169dc1"
         ],
@@ -764,6 +788,7 @@
         "related.hash": [
             "6144:+60EDP6uCLfGw/GpxXinM1BCo1PlumGx2mx2tXd0t115JGz:X5DpBw/KViMTB1MnEWk0115JU",
             "68aea345b134d576ccdef7f06db86088",
+            "9244D022AD13DD37E1F400FCA6A58F8561626E381F00A89777D41F8A98356F1BB2B717",
             "fb25d13188a5d0913bbcf5aeff6c7e3208ad92a7d10ab6bed2735f4d43310a27",
             "ff60107d82dcda7e6726d214528758e7"
         ],
@@ -782,4 +807,4 @@
         "threatintel.indicator.first_seen": "2021-01-14T06:04:20.000Z",
         "threatintel.indicator.type": "file"
     }
-]
+]