From e7f83c31dcf0413936406377dcbd030e709dba02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20Ruiz?= Date: Wed, 4 Sep 2024 13:22:10 +0200 Subject: [PATCH] Remove unused fields from the vulnerabilities index template (#369) * Remove unused fields from the vulnerabilities index template * Fix and use remove_multi_fields function --- ecs/generate.sh | 28 ++++++++++++++- .../event-generator/event_generator.py | 36 ++++++------------- .../fields/custom/wazuh.yml | 5 --- ecs/vulnerability-detector/fields/subset.yml | 9 +++-- .../fields/template-settings-legacy.json | 7 ++-- .../fields/template-settings.json | 7 ++-- 6 files changed, 51 insertions(+), 41 deletions(-) diff --git a/ecs/generate.sh b/ecs/generate.sh index 4b747c0c9a0cb..f40a7d824e348 100755 --- a/ecs/generate.sh +++ b/ecs/generate.sh @@ -13,6 +13,19 @@ show_usage() { echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200" } +# Function to remove multi-fields from the generated index template +remove_multi_fields() { + local IN_FILE="$1" + local OUT_FILE="$2" + + jq 'del( + .mappings.properties.host.properties.os.properties.full.fields, + .mappings.properties.host.properties.os.properties.name.fields, + .mappings.properties.vulnerability.properties.description.fields + )' "$IN_FILE" > "$OUT_FILE" +} + + # Function to generate mappings generate_mappings() { local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields" @@ -34,8 +47,21 @@ generate_mappings() { echo "Replacing \"match_only_text\" type with \"text\"" find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \; + local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json" + local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json" + + # Delete the "tags" field from the index template + echo "Deleting the \"tags\" field from the index template" + jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE" + mv "$OUT_FILE" "$IN_FILE" + + # Remove multi-fields from the generated index template + echo "Removing multi-fields from the index template" + remove_multi_fields "$IN_FILE" "$OUT_FILE" + mv "$OUT_FILE" "$IN_FILE" + # Transform legacy index template for OpenSearch compatibility - cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{ + cat "$IN_FILE" | jq '{ "index_patterns": .index_patterns, "priority": .order, "template": { diff --git a/ecs/vulnerability-detector/event-generator/event_generator.py b/ecs/vulnerability-detector/event-generator/event_generator.py index 73f8e8be71205..0b95ac22d7f62 100755 --- a/ecs/vulnerability-detector/event-generator/event_generator.py +++ b/ecs/vulnerability-detector/event-generator/event_generator.py @@ -2,15 +2,13 @@ # This script generates sample events and injects them into the Wazuh Indexer. # The events follow the Elastic Common Schema (ECS) format, and contains the following fields: -# - ecs -# - base -# - event # - agent # - package # - host # - vulnerability +# - wazuh (custom) # -# This is an ad-hoc script for the vulnearbility module. Extend to support other modules. +# This is an ad-hoc script for the vulnerability module. Extend to support other modules. import datetime import random @@ -102,7 +100,6 @@ def generate_random_host(): version = f'{random.randint(0, 99)}.{random.randint(0, 99)}' host = { 'os': { - 'family': family, 'full': f'{family} {version}', 'kernel': f'{version}kernel{random.randint(0, 99)}', 'name': f'{family} {version}', @@ -141,11 +138,6 @@ def generate_random_package(): return package -def generate_random_tags(): - tags = [f'tag{random.randint(0, 99)}' for _ in range(random.randint(0, 9))] - return tags - - def generate_random_vulnerability(): id = random.randint(0, 9999) vulnerability = { @@ -176,9 +168,9 @@ def generate_random_wazuh(): 'name': f'wazuh-cluster-{random.randint(0,10)}', 'node': f'wazuh-cluster-node-{random.randint(0,10)}' }, - 'manager': { - 'name': f'wazuh-manager-{random.randint(0,10)}' - }, + # 'manager': { + # 'name': f'wazuh-manager-{random.randint(0,10)}' + # }, 'schema': { 'version': '1.7.0' }, @@ -190,15 +182,9 @@ def generate_random_data(number): data = [] for _ in range(number): event_data = { - # '@timestamp': generate_random_date(), 'agent': generate_random_agent(), - # 'ecs': {'version': '1.7.0'}, - # 'event': generate_random_event(), 'host': generate_random_host(), - # 'labels': generate_random_labels(), - 'message': f'message{random.randint(0, 99999)}', 'package': generate_random_package(), - 'tags': generate_random_tags(), 'vulnerability': generate_random_vulnerability(), 'wazuh': generate_random_wazuh() } @@ -227,7 +213,7 @@ def inject_events(ip, port, index, username, password, data): def main(): try: - number = int(input("How many events do you want to generate? ")) + number = int(input("How many events do you want to generate? ").strip() or 50) except ValueError: logging.error("Invalid input. Please enter a valid number.") return @@ -245,11 +231,11 @@ def main(): inject = input( "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower() if inject == 'y': - ip = input("Enter the IP of your Indexer: ") - port = input("Enter the port of your Indexer: ") - index = input("Enter the index name: ") - username = input("Username: ") - password = input("Password: ") + ip = input("Enter the IP of your Indexer: ").strip() or "localhost" + port = input("Enter the port of your Indexer: ").strip() or 9200 + index = input("Enter the index name: ").strip() or "wazuh-states-vulnerability-test" + username = input("Username: ").strip() or "admin" + password = input("Password: ").strip() inject_events(ip, port, index, username, password, data) diff --git a/ecs/vulnerability-detector/fields/custom/wazuh.yml b/ecs/vulnerability-detector/fields/custom/wazuh.yml index 235a746758812..cbc1a38f016df 100644 --- a/ecs/vulnerability-detector/fields/custom/wazuh.yml +++ b/ecs/vulnerability-detector/fields/custom/wazuh.yml @@ -14,11 +14,6 @@ level: custom description: > Wazuh cluster node name. - - name: manager.name - type: keyword - level: custom - description: > - Wazuh manager name. Used by dashboards to filter results on single node deployments. - name: schema.version type: keyword level: custom diff --git a/ecs/vulnerability-detector/fields/subset.yml b/ecs/vulnerability-detector/fields/subset.yml index 951f7e492d1c2..e917ab119bff5 100644 --- a/ecs/vulnerability-detector/fields/subset.yml +++ b/ecs/vulnerability-detector/fields/subset.yml @@ -4,7 +4,6 @@ fields: base: fields: tags: [] - message: "" agent: fields: "*" package: @@ -12,7 +11,13 @@ fields: host: fields: os: - fields: "*" + fields: + full: "" + kernel: "" + name: "" + platform: "" + type: "" + version: "" vulnerability: fields: "*" wazuh: diff --git a/ecs/vulnerability-detector/fields/template-settings-legacy.json b/ecs/vulnerability-detector/fields/template-settings-legacy.json index 205850de68e83..22ac9ab3507ea 100644 --- a/ecs/vulnerability-detector/fields/template-settings-legacy.json +++ b/ecs/vulnerability-detector/fields/template-settings-legacy.json @@ -1,6 +1,6 @@ { "index_patterns": [ - "wazuh-states-vulnerabilities" + "wazuh-states-vulnerabilities-*" ], "order": 1, "settings": { @@ -15,15 +15,14 @@ "number_of_replicas": "0", "refresh_interval": "2s", "query.default_field": [ - "base.tags", "agent.id", "host.os.family", - "host.os.full.text", + "host.os.full", "host.os.version", "package.name", "package.version", "vulnerability.id", - "vulnerability.description.text", + "vulnerability.description", "vulnerability.severity", "wazuh.cluster.name" ] diff --git a/ecs/vulnerability-detector/fields/template-settings.json b/ecs/vulnerability-detector/fields/template-settings.json index 96fb7712bf3bb..4d347360af99b 100644 --- a/ecs/vulnerability-detector/fields/template-settings.json +++ b/ecs/vulnerability-detector/fields/template-settings.json @@ -1,6 +1,6 @@ { "index_patterns": [ - "wazuh-states-vulnerabilities" + "wazuh-states-vulnerabilities-*" ], "priority": 1, "template": { @@ -16,15 +16,14 @@ "number_of_replicas": "0", "refresh_interval": "2s", "query.default_field": [ - "base.tags", "agent.id", "host.os.family", - "host.os.full.text", + "host.os.full", "host.os.version", "package.name", "package.version", "vulnerability.id", - "vulnerability.description.text", + "vulnerability.description", "vulnerability.severity", "wazuh.cluster.name" ]