Moving logstash dockerfile and associated files to its own folder

integrations/.gitignore

@@ -1,2 +1,3 @@
 external
-docker/certs
+docker/certs
+docker/config

integrations/amazon-security-lake/logstash/pipeline/indexer-to-s3.conf

@@ -27,19 +27,27 @@ output {
    s3 {
       id => "output.s3"
       access_key_id => "${AWS_ACCESS_KEY_ID}"
-      secret_access_key => "${AWS_SECRET_ACCESS_KEY}"
-      region => "${AWS_REGION}"
-      endpoint => "${AWS_ENDPOINT}"
       bucket => "${S3_BUCKET_RAW}"
       codec => "json_lines"
-      retry_count => 0
-      validate_credentials_on_root_bucket => false
+      encoding => "gzip"
+      endpoint => "${AWS_ENDPOINT}"
       prefix => "%{+YYYY}%{+MM}%{+dd}"
+      region => "${AWS_REGION}"
+      retry_count => 0
+      secret_access_key => "${AWS_SECRET_ACCESS_KEY}"
       server_side_encryption => true
       server_side_encryption_algorithm => "AES256"
+      time_file => 5
+      validate_credentials_on_root_bucket => false
       additional_settings => {
          "force_path_style" => true
       }
-      time_file => 5
+   }
+   file {
+      id => "output.file"
+      path => "/usr/share/logstash/logs/indexer-to-file-%{+YYYY-MM-dd-HH}.log"
+      file_mode => 0644
+      codec => json_lines
+      flush_interval => 30
    }
 }

integrations/amazon-security-lake/src/lambda_function.py

@@ -2,6 +2,7 @@
 import os
 import urllib.parse
 import json
+import gzip
 import boto3
 import pyarrow as pa
 import pyarrow.parquet as pq
@@ -31,7 +32,7 @@ def get_events(bucket: str, key: str) -> list:
     logger.info(f"Reading {key}.")
     try:
         response = s3_client.get_object(Bucket=bucket, Key=key)
-        data = response['Body'].read().decode('utf-8')
+        data = gzip.decompress(response['Body'].read()).decode('utf-8')
         return data.splitlines()
     except ClientError as e:
         logger.error(

integrations/docker/.env

@@ -4,9 +4,6 @@ ELASTIC_PASSWORD=elastic
 # Password for the 'kibana_system' user (at least 6 characters)
 KIBANA_PASSWORD=elastic
 
-# Version of Elastic products
-STACK_VERSION=8.6.2
-
 # Set the cluster name
 CLUSTER_NAME=elastic
 
@@ -22,8 +19,26 @@ KIBANA_PORT=5602
 # Increase or decrease based on the available host memory (in bytes)
 MEM_LIMIT=1073741824
 
+# Wazuh version
+WAZUH_VERSION=4.7.5
+
+# Wazuh Indexer version (Provisionally using OpenSearch)
+WAZUH_INDEXER_VERSION=2.14.0
+
+# Wazuh Dashboard version (Provisionally using OpenSearch Dashboards)
+WAZUH_DASHBOARD_VERSION=2.14.0
+
+# Wazuh certs generator version
+WAZUH_CERTS_GENERATOR_VERSION=0.0.1
+
 # OpenSearch destination cluster version
 OS_VERSION=2.14.0
 
-# Wazuh version
-WAZUH_VERSION=4.7.5
+# Logstash version:
+LOGSTASH_OSS_VERSION=8.9.0
+
+# Splunk version:
+SPLUNK_VERSION=9.1.4
+
+# Version of Elastic products
+STACK_VERSION=8.14.3

integrations/docker/compose.amazon-security-lake.yml

@@ -13,7 +13,7 @@ services:
     command: bash -c "python run.py -a wazuh.indexer"
 
   wazuh.indexer:
-    image: opensearchproject/opensearch:2.12.0
+    image: opensearchproject/opensearch:${WAZUH_INDEXER_VERSION}
     container_name: wazuh.indexer
     depends_on:
       wazuh-certs-generator:
@@ -22,6 +22,7 @@ services:
     ports:
       - 9200:9200
     environment:
+      - WAZUH_INDEXER_VERSION=${WAZUH_INDEXER_VERSION}
       # - cluster.name=opensearch-cluster
       - node.name=wazuh.indexer
       - discovery.type=single-node
@@ -56,7 +57,7 @@ services:
       - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
 
   wazuh.dashboard:
-    image: opensearchproject/opensearch-dashboards:2.12.0
+    image: opensearchproject/opensearch-dashboards:${WAZUH_DASHBOARD_VERSION}
     container_name: wazuh.dashboard
     depends_on:
       - wazuh.indexer
@@ -71,6 +72,7 @@ services:
       - ./certs/wazuh.dashboard.pem:/usr/share/opensearch-dashboards/config/certs/opensearch.pem
       - ./certs/root-ca.pem:/usr/share/opensearch-dashboards/config/certs/root-ca.pem
     environment:
+      WAZUH_DASHBOARD_VERSION: ${WAZUH_DASHBOARD_VERSION}
       OPENSEARCH_HOSTS: '["https://wazuh.indexer:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query
       SERVER_SSL_ENABLED: 'true'
       SERVER_SSL_KEY: '/usr/share/opensearch-dashboards/config/certs/opensearch.key'
@@ -80,7 +82,9 @@ services:
   wazuh.integration.security.lake:
     image: wazuh/indexer-security-lake-integration
     build:
-      context: ../amazon-security-lake
+      context: ../logstash
+      args:
+        - LOGSTASH_OSS_VERSION=${LOGSTASH_OSS_VERSION}
     container_name: wazuh.integration.security.lake
     depends_on:
       - wazuh.indexer
@@ -104,7 +108,6 @@ services:
       - ./certs/root-ca.pem:/usr/share/logstash/root-ca.pem
       - ../amazon-security-lake/src:/usr/share/logstash/amazon-security-lake # TODO use dedicated folder
       # - ./credentials:/usr/share/logstash/.aws/credentials # TODO credentials are not commited (missing)
-    command: tail -f /var/log/logstash/logstash-plain.log
 
   s3.ninja:
     image: scireum/s3-ninja:latest
@@ -126,6 +129,7 @@ services:
       AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE"
       AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
       AWS_REGION: "us-east-1"
+      REGION: "us-east-1"
       AWS_BUCKET: "wazuh-aws-security-lake-parquet"
       S3_BUCKET_OCSF: "wazuh-aws-security-lake-ocsf"
       AWS_ENDPOINT: "http://s3.ninja:9000"
@@ -159,12 +163,14 @@ services:
       "
 
   wazuh-certs-generator:
-    image: wazuh/wazuh-certs-generator:0.0.1
+    image: wazuh/wazuh-certs-generator:${WAZUH_CERTS_GENERATOR_VERSION}
     hostname: wazuh-certs-generator
     depends_on:
       generate-certs-config:
         condition: service_completed_successfully
     container_name: wazuh-certs-generator
+    environment:
+      - WAZUH_CERTS_GENERATOR_VERSION=${WAZUH_CERTS_GENERATOR_VERSION}
     entrypoint: sh -c "/entrypoint.sh; chown -R 1000:999 /certificates; chmod 740 /certificates; chmod 440 /certificates/*"
     volumes:
       - ./certs/:/certificates/

integrations/docker/compose.indexer-elastic.yml

@@ -11,14 +11,15 @@ services:
     command: bash -c "python run.py -a wazuh.indexer"
 
   wazuh.indexer:
-    image: opensearchproject/opensearch:2.12.0
+    image: opensearchproject/opensearch:${WAZUH_INDEXER_VERSION}
     depends_on:
       wazuh-certs-generator:
         condition: service_completed_successfully
     hostname: wazuh.indexer
     ports:
       - 9200:9200
     environment:
+      - WAZUH_INDEXER_VERSION=${WAZUH_INDEXER_VERSION}
       - node.name=wazuh.indexer
       - discovery.type=single-node
       - bootstrap.memory_lock=true
@@ -51,7 +52,7 @@ services:
       - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
 
   wazuh.dashboard:
-    image: opensearchproject/opensearch-dashboards:2.12.0
+    image: opensearchproject/opensearch-dashboards:${WAZUH_DASHBOARD_VERSION}
     depends_on:
       - wazuh.indexer
     hostname: wazuh.dashboard
@@ -65,6 +66,7 @@ services:
       - ./certs/wazuh.dashboard.pem:/usr/share/opensearch-dashboards/config/certs/opensearch.pem
       - ./certs/root-ca.pem:/usr/share/opensearch-dashboards/config/certs/root-ca.pem
     environment:
+      WAZUH_DASHBOARD_VERSION: ${WAZUH_DASHBOARD_VERSION}
       OPENSEARCH_HOSTS: '["https://wazuh.indexer:9200"]' # Define the OpenSearch nodes that OpenSearch Dashboards will query
       SERVER_SSL_ENABLED: 'true'
       SERVER_SSL_KEY: '/usr/share/opensearch-dashboards/config/certs/opensearch.key'
@@ -92,8 +94,10 @@ services:
       "
 
   wazuh-certs-generator:
-    image: wazuh/wazuh-certs-generator:0.0.1
+    image: wazuh/wazuh-certs-generator:${WAZUH_CERTS_GENERATOR_VERSION}
     hostname: wazuh-certs-generator
+    environment:
+      - WAZUH_CERTS_GENERATOR_VERSION=${WAZUH_CERTS_GENERATOR_VERSION}
     depends_on:
       generate-certs-config:
         condition: service_completed_successfully
@@ -237,9 +241,11 @@ services:
     depends_on:
       es01:
         condition: service_healthy
-    image: logstash-oss:8.6.2
+    image: logstash-oss:${LOGSTASH_OSS_VERSION}
     build:
-      context: ../elastic
+      context: ../logstash
+      args:
+        - LOGSTASH_OSS_VERSION=${LOGSTASH_OSS_VERSION}
     environment:
       LOG_LEVEL: info
       MONITORING_ENABLED: false

integrations/docker/compose.indexer-opensearch.yml

@@ -11,14 +11,15 @@ services:
     command: bash -c "python run.py -a wazuh.indexer"
 
   wazuh.indexer:
-    image: opensearchproject/opensearch:2.12.0
+    image: opensearchproject/opensearch:${WAZUH_INDEXER_VERSION}
     depends_on:
       wazuh-certs-generator:
         condition: service_completed_successfully
     hostname: wazuh.indexer
     ports:
       - 9200:9200
     environment:
+      - WAZUH_INDEXER_VERSION=${WAZUH_INDEXER_VERSION}
       - node.name=wazuh.indexer
       - discovery.type=single-node
       - bootstrap.memory_lock=true
@@ -51,7 +52,7 @@ services:
       - ./certs/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
 
   wazuh.dashboard:
-    image: opensearchproject/opensearch-dashboards:2.12.0
+    image: opensearchproject/opensearch-dashboards:${WAZUH_DASHBOARD_VERSION}
     depends_on:
       - wazuh.indexer
     hostname: wazuh.dashboard
@@ -66,6 +67,7 @@ services:
       - ./certs/opensearch.dashboards.pem:/usr/share/opensearch-dashboards/config/certs/opensearch.pem
       - ./certs/root-ca.pem:/usr/share/opensearch-dashboards/config/certs/root-ca.pem
     environment:
+      WAZUH_DASHBOARD_VERSION: ${WAZUH_DASHBOARD_VERSION}
       OPENSEARCH_HOSTS: '["https://wazuh.indexer:9200"]'
       SERVER_SSL_ENABLED: 'true'
       SERVER_SSL_KEY: '/usr/share/opensearch-dashboards/config/certs/opensearch.key'
@@ -97,8 +99,10 @@ services:
       "
 
   wazuh-certs-generator:
-    image: wazuh/wazuh-certs-generator:0.0.1
+    image: wazuh/wazuh-certs-generator:${WAZUH_CERTS_GENERATOR_VERSION}
     hostname: wazuh-certs-generator
+    environment:
+      - WAZUH_CERTS_GENERATOR_VERSION=${WAZUH_CERTS_GENERATOR_VERSION}
     depends_on:
       generate-certs-config:
         condition: service_completed_successfully
@@ -169,13 +173,16 @@ services:
       - 'OPENSEARCH_HOSTS="https://opensearch.node:9200"'
 
   logstash:
-    image: logstash-oss:8.6.2
+    image: logstash-oss:${LOGSTASH_OSS_VERSION}
     depends_on:
       opensearch.node:
         condition: service_healthy
     build:
-      context: ../opensearch
+      context: ../logstash
+      args:
+        - LOGSTASH_OSS_VERSION=${LOGSTASH_OSS_VERSION}
     environment:
+      STACK_VERSION: ${STACK_VERSION}
       LOG_LEVEL: info
       MONITORING_ENABLED: false
     volumes: