Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[filebeat][threatintel] MISP splitting fix for empty responses #38917

Merged
merged 5 commits into from
Apr 15, 2024
Merged

[filebeat][threatintel] MISP splitting fix for empty responses #38917

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3d605cd
Set response.split.ignore_empty_value to true to avoid processing the…
chrisberkhout Apr 13, 2024
86f5c7c
Correct null checking.
chrisberkhout Apr 13, 2024
fc53bb7
Additional fixes to null-safe operator usage.
chrisberkhout Apr 13, 2024
1ee99a4
Update CHANGELOG.next.asciidoc.
chrisberkhout Apr 13, 2024
cdb10b4
Update CHANGELOG.next.asciidoc
chrisberkhout Apr 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
- Fix handling of truncated files in Filestream {issue}38070[38070] {pull}38416[38416]
- Fix panic when more than 32767 pipeline clients are active. {issue}38197[38197] {pull}38556[38556]
- Fix filestream's registry GC: registry entries are now removed from the in-memory and disk store when they're older than the set TTL {issue}36761[36761] {pull}38488[38488]

- [threatintel] MISP splitting fix for empty responses {issue}38739[38739] {pull}38917[38917]

*Heartbeat*

Expand Down
1 change: 1 addition & 0 deletions x-pack/filebeat/module/threatintel/misp/config/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ request.transforms:

response.split:
target: body.response
ignore_empty_value: true
split:
target: body.Event.Attribute
ignore_empty_value: true
Expand Down
36 changes: 18 additions & 18 deletions x-pack/filebeat/module/threatintel/misp/ingest/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,12 +63,12 @@ processors:
field:
- misp.Attribute
ignore_missing: true
if: ctx.misp?.Attribute.size() == 0
if: ctx.misp?.Attribute?.size() == 0
- remove:
field:
- misp.Object
ignore_missing: true
if: ctx.misp?.Object.size() == 0
if: ctx.misp?.Object?.size() == 0
- date:
field: misp.timestamp
formats:
Expand Down Expand Up @@ -144,12 +144,12 @@ processors:
- set:
field: threat.indicator.type
value: file
if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type?.startsWith('filename') == true"
- rename:
field: misp.attribute.value
target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
ignore_missing: true
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')"
- rename:
field: misp.attribute.value
target_field: threat.indicator.file.name
Expand All @@ -160,23 +160,23 @@ processors:
patterns:
- "%{WORD}\\|%{WORD:_tmp.hashtype}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
- grok:
field: misp.attribute.value
patterns:
- "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
ignore_missing: true
if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
if: ctx.misp?.attribute?.type?.startsWith('filename|') == true
- set:
field: threat.indicator.file.hash.{{_tmp.hashtype}}
value: "{{_tmp.hashvalue}}"
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"
if: "ctx.misp?.attribute?.type?.startsWith('filename|') == true && ctx._tmp?.hashvalue != null && ctx._tmp?.hashtype != null"

## URL/URI indicator operations
- set:
field: threat.indicator.type
value: url
if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
if: "['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
- uri_parts:
field: misp.attribute.value
target_field: threat.indicator.url
Expand All @@ -193,7 +193,7 @@ processors:
- set:
field: threat.indicator.type
value: windows-registry-key
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
if: "ctx.misp?.attribute?.type?.startsWith('regkey') == true"
- rename:
field: misp.attribute.value
target_field: threat.indicator.registry.key
Expand All @@ -210,7 +210,7 @@ processors:
- set:
field: threat.indicator.type
value: autonomous-system
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
if: "ctx.misp?.attribute?.type == 'AS'"
- convert:
field: misp.attribute.value
type: long
Expand All @@ -222,11 +222,11 @@ processors:
- set:
field: threat.indicator.type
value: domain-name
if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
if: "ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type?.startsWith('domain') == true"
- set:
field: threat.indicator.type
value: ipv4-addr
if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
if: "['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.url.domain
Expand Down Expand Up @@ -257,11 +257,11 @@ processors:
- set:
field: threat.indicator.type
value: email-addr
if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
if: "['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
- set:
field: threat.indicator.type
value: email-message
if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
if: "ctx.misp?.attribute?.type?.startsWith('email') == true && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.email.address
Expand All @@ -274,13 +274,13 @@ processors:
- append:
field: user.roles
value: "reporting_user"
if: ctx?.user?.email != null
if: ctx.user?.email != null

## MAC Address indicator operations
- set:
field: threat.indicator.type
value: mac-addr
if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
if: "['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
- rename:
field: misp.attribute.value
target_field: threat.indicator.mac
Expand Down Expand Up @@ -337,12 +337,12 @@ processors:
######################
- remove:
field: event.original
if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))"
ignore_failure: true
ignore_missing: true
- script:
lang: painless
if: ctx?.misp != null
if: ctx.misp != null
source: |
void handleMap(Map map) {
for (def x : map.values()) {
Expand Down
Loading