Remove unused fields from the vulnerabilities index template (#369)

* Remove unused fields from the vulnerabilities index template

* Fix and use remove_multi_fields function

ecs/generate.sh

@@ -13,6 +13,19 @@ show_usage() {
   echo "Example: $0 v8.10.0 ~/wazuh-indexer vulnerability-detector --upload https://indexer:9200"
 }
 
+# Function to remove multi-fields from the generated index template
+remove_multi_fields() {
+  local IN_FILE="$1"
+  local OUT_FILE="$2"
+
+  jq 'del(
+    .mappings.properties.host.properties.os.properties.full.fields,
+    .mappings.properties.host.properties.os.properties.name.fields,
+    .mappings.properties.vulnerability.properties.description.fields
+  )' "$IN_FILE" > "$OUT_FILE"
+}
+
+
 # Function to generate mappings
 generate_mappings() {
   local IN_FILES_DIR="$INDEXER_SRC/ecs/$MODULE/fields"
@@ -34,8 +47,21 @@ generate_mappings() {
   echo "Replacing \"match_only_text\" type with \"text\""
   find "$OUT_DIR" -type f -exec sed -i 's/match_only_text/text/g' {} \;
 
+  local IN_FILE="$OUT_DIR/generated/elasticsearch/legacy/template.json"
+  local OUT_FILE="$OUT_DIR/generated/elasticsearch/legacy/template-tmp.json"
+
+  # Delete the "tags" field from the index template
+  echo "Deleting the \"tags\" field from the index template"
+  jq 'del(.mappings.properties.tags)' "$IN_FILE" > "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
+  # Remove multi-fields from the generated index template
+  echo "Removing multi-fields from the index template"
+  remove_multi_fields "$IN_FILE" "$OUT_FILE"
+  mv "$OUT_FILE" "$IN_FILE"
+
   # Transform legacy index template for OpenSearch compatibility
-  cat "$OUT_DIR/generated/elasticsearch/legacy/template.json" | jq '{
+  cat "$IN_FILE" | jq '{
     "index_patterns": .index_patterns,
     "priority": .order,
     "template": {

ecs/vulnerability-detector/event-generator/event_generator.py

@@ -2,15 +2,13 @@
 
 # This script generates sample events and injects them into the Wazuh Indexer.
 # The events follow the Elastic Common Schema (ECS) format, and contains the following fields:
-#   - ecs
-#   - base
-#   - event
 #   - agent
 #   - package
 #   - host
 #   - vulnerability
+#   - wazuh (custom)
 #
-# This is an ad-hoc script for the vulnearbility module. Extend to support other modules.
+# This is an ad-hoc script for the vulnerability module. Extend to support other modules.
 
 import datetime
 import random
@@ -102,7 +100,6 @@ def generate_random_host():
     version = f'{random.randint(0, 99)}.{random.randint(0, 99)}'
     host = {
         'os': {
-            'family': family,
             'full': f'{family} {version}',
             'kernel': f'{version}kernel{random.randint(0, 99)}',
             'name': f'{family} {version}',
@@ -141,11 +138,6 @@ def generate_random_package():
     return package
 
 
-def generate_random_tags():
-    tags = [f'tag{random.randint(0, 99)}' for _ in range(random.randint(0, 9))]
-    return tags
-
-
 def generate_random_vulnerability():
     id = random.randint(0, 9999)
     vulnerability = {
@@ -176,9 +168,9 @@ def generate_random_wazuh():
             'name': f'wazuh-cluster-{random.randint(0,10)}',
             'node': f'wazuh-cluster-node-{random.randint(0,10)}'
         },
-        'manager': {
-            'name': f'wazuh-manager-{random.randint(0,10)}'
-        },
+        # 'manager': {
+        #     'name': f'wazuh-manager-{random.randint(0,10)}'
+        # },
         'schema': {
             'version': '1.7.0'
         },
@@ -190,15 +182,9 @@ def generate_random_data(number):
     data = []
     for _ in range(number):
         event_data = {
-            # '@timestamp': generate_random_date(),
             'agent': generate_random_agent(),
-            # 'ecs': {'version': '1.7.0'},
-            # 'event': generate_random_event(),
             'host': generate_random_host(),
-            # 'labels': generate_random_labels(),
-            'message': f'message{random.randint(0, 99999)}',
             'package': generate_random_package(),
-            'tags': generate_random_tags(),
             'vulnerability': generate_random_vulnerability(),
             'wazuh': generate_random_wazuh()
         }
@@ -227,7 +213,7 @@ def inject_events(ip, port, index, username, password, data):
 
 def main():
     try:
-        number = int(input("How many events do you want to generate? "))
+        number = int(input("How many events do you want to generate? ").strip() or 50)
     except ValueError:
         logging.error("Invalid input. Please enter a valid number.")
         return
@@ -245,11 +231,11 @@ def main():
     inject = input(
         "Do you want to inject the generated data into your indexer? (y/n) ").strip().lower()
     if inject == 'y':
-        ip = input("Enter the IP of your Indexer: ")
-        port = input("Enter the port of your Indexer: ")
-        index = input("Enter the index name: ")
-        username = input("Username: ")
-        password = input("Password: ")
+        ip = input("Enter the IP of your Indexer: ").strip() or "localhost"
+        port = input("Enter the port of your Indexer: ").strip() or 9200
+        index = input("Enter the index name: ").strip() or "wazuh-states-vulnerability-test"
+        username = input("Username: ").strip() or "admin"
+        password = input("Password: ").strip()
         inject_events(ip, port, index, username, password, data)
 
 

ecs/vulnerability-detector/fields/custom/wazuh.yml

@@ -14,11 +14,6 @@
       level: custom
       description: >
         Wazuh cluster node name.
-    - name: manager.name
-      type: keyword
-      level: custom
-      description: >
-        Wazuh manager name. Used by dashboards to filter results on single node deployments.
     - name: schema.version
       type: keyword
       level: custom

ecs/vulnerability-detector/fields/subset.yml

@@ -4,15 +4,20 @@ fields:
   base:
     fields: 
       tags: []
-      message: ""
   agent:
     fields: "*"
   package:
     fields: "*"
   host:
     fields:
       os:
-        fields: "*"
+        fields: 
+          full: ""
+          kernel: ""
+          name: ""
+          platform: ""
+          type: ""
+          version: ""
   vulnerability:
     fields: "*"
   wazuh:

ecs/vulnerability-detector/fields/template-settings-legacy.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": [
-    "wazuh-states-vulnerabilities"
+    "wazuh-states-vulnerabilities-*"
   ],
   "order": 1,
   "settings": {
@@ -15,15 +15,14 @@
       "number_of_replicas": "0",
       "refresh_interval": "2s",
       "query.default_field": [
-        "base.tags",
         "agent.id",
         "host.os.family",
-        "host.os.full.text",
+        "host.os.full",
         "host.os.version",
         "package.name",
         "package.version",
         "vulnerability.id",
-        "vulnerability.description.text",
+        "vulnerability.description",
         "vulnerability.severity",
         "wazuh.cluster.name"
       ]

ecs/vulnerability-detector/fields/template-settings.json

@@ -1,6 +1,6 @@
 {
   "index_patterns": [
-    "wazuh-states-vulnerabilities"
+    "wazuh-states-vulnerabilities-*"
   ],
   "priority": 1,
   "template": {
@@ -16,15 +16,14 @@
         "number_of_replicas": "0",
         "refresh_interval": "2s",
         "query.default_field": [
-          "base.tags",
           "agent.id",
           "host.os.family",
-          "host.os.full.text",
+          "host.os.full",
           "host.os.version",
           "package.name",
           "package.version",
           "vulnerability.id",
-          "vulnerability.description.text",
+          "vulnerability.description",
           "vulnerability.severity",
           "wazuh.cluster.name"
         ]