E2E tests: Add an initial validation stage

tests/end_to_end/README.md

@@ -18,6 +18,7 @@ Our E2E tests will verify that, after generating an event, an alert will be trig
 To run these tests we need to use a **Linux** machine and install the following tools:
 
 - [Ansible](https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html)
+- [netcat](https://en.wikipedia.org/wiki/Netcat)
 
 Then, you will need to have an inventory with the needed hosts and variables. For example:
 
@@ -155,6 +156,39 @@ To execute these tests, we need to run the following command:
 python -m pytest <TEST_PATH> --inventory_path=<INVENTORY_PATH>
 ```
 
+### Adding or modifying E2E tests
+
+When adding or modifying any test it is necesry to modify the file with the environment data, placed in `tests/end_to_end/data/env_requirements.json`
+
+This file is used to validate the environments where the selected tests will be executed and it follows this structure:
+```
+"<TEST_SUITE_NAME>": {
+    "manager": {
+        "instances": <QUANTITY_OF_INSTANCES (any positive number)>,
+        "distros": <SUPPORTED_DISTROS (list)>
+    },
+    "agent": {
+        "instances": <QUANTITY_OF_INSTANCES (any positive number)>,
+        "distros": <SUPPORTED_DISTROS (list)>
+    }
+}
+```
+
+### Add specific validation tasks (for a test module)
+
+To add specific validation tasks to a test, its necessary to add a new playbook inside the test module, in the playbook folder with the default Play structure:
+
+```
+- name: <ANY-NAME>
+  hosts: "{{ target_hosts }}"
+  tasks:
+    <ANY-TASK-YOU-WANT>
+```
+
+E.g: Add validation tasks for test_audit by creating a playbook called `validation.yaml` in `tests/end_to_end/test_basic_cases/test_audit/data/playbooks`
+
+> The file name must always be "validation.yaml"
+
 #### Audit tests examples
 
 ```shell script

tests/end_to_end/conftest.py

@@ -4,13 +4,144 @@
 import os
 import ansible_runner
 import pytest
+import json
 from tempfile import gettempdir
 
 from wazuh_testing.tools.file import remove_file
 from wazuh_testing import end_to_end as e2e
 
 
 alerts_json = os.path.join(gettempdir(), 'alerts.json')
+suite_path = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_target_hosts_and_distros(test_suite_name, target_distros=[], target_hosts=[]):
+    environment_file = os.path.join(suite_path, 'data', 'env_requirements.json')
+    environment_metadata = json.load(open(environment_file))
+    distros_by = {'manager': [], 'agent': []}
+
+    for key in environment_metadata[test_suite_name]:
+        if environment_metadata[test_suite_name][key]['instances'] > 0:
+            # Save manager/agent distros
+            distros_by[key] = environment_metadata[test_suite_name][key]['distros']
+            target_distros.extend(environment_metadata[test_suite_name][key]['distros'])
+            # Add the target host to the list (following the standard host name: "<distro>-<type>*")
+            target_hosts.extend([distro.lower() + f"-{key}" for distro in distros_by[key]])
+    # Remove duplicates
+    target_hosts = list(dict.fromkeys(target_hosts))
+    target_distros = list(dict.fromkeys(target_distros))
+
+    return target_hosts, target_distros
+
+
+@pytest.fixture(scope='session', autouse=True)
+def validate_environments(request):
+    """Fixture with session scope to validate the environments before run the E2E tests.
+
+    This phase is divided into 4 steps:
+        Step 1: Collect the data related to the selected tests that will be executed.
+        Step 2: Generate a playbook containing cross-checks for selected tests.
+        Step 3: Run the generated playbook.
+
+    Args:
+        request (fixture):  Gives access to the requesting test context.
+    """
+    collected_items = request.session.items
+    roles_path = request.config.getoption('--roles-path')
+    inventory_path = request.config.getoption('--inventory_path')
+    playbook_generator = os.path.join(suite_path, 'data', 'validation_playbooks', 'generate_general_play.yaml')
+    playbook_template = os.path.join(suite_path, 'data', 'validation_templates', 'general_validation.j2')
+    general_playbook = os.path.join(suite_path, 'data', 'validation_playbooks', 'general_validation.yaml')
+
+    if not inventory_path:
+        raise ValueError('Inventory not specified')
+
+    # --------------------------------------- Step 1: Prepare the necessary data ---------------------------------------
+    test_suites_paths = []
+    target_hosts = []
+    target_distros = []
+
+    # Get the path of the tests from collected items.
+    collected_paths = [item.fspath for item in collected_items]
+    # Remove duplicates caused by the existence of 2 or more test cases
+    collected_paths = list(dict.fromkeys(collected_paths))
+
+    for path in collected_paths:
+        # Remove the name of the file from the path
+        path = str(path).rsplit('/', 1)[0]
+        # Add the test suite path
+        test_suites_paths.append(path)
+        # Get the test suite name
+        test_suite_name = path.split('/')[-1:][0]
+        # Set target hosts and distros
+        target_hosts, target_distros = get_target_hosts_and_distros(test_suite_name, target_distros, target_hosts)
+    # -------------------------------------------------- End of Step 1 -------------------------------------------------
+
+    # ---------------------- Step 2: Run the playbook to generate the general validation playbook ----------------------
+    gen_parameters = {
+        'playbook': playbook_generator, 'inventory': inventory_path,
+        'extravars': {
+            'template_path': playbook_template,
+            'dest_path': general_playbook,
+            'target_hosts': ','.join(target_hosts),
+            'distros': target_distros
+        }
+    }
+    ansible_runner.run(**gen_parameters)
+    # -------------------------------------------------- End of Step 2 -------------------------------------------------
+
+    # ----------------------------------- Step 3: Run the general validation playbook ----------------------------------
+    parameters = {
+        'playbook': general_playbook,
+        'inventory': inventory_path,
+        'envvars': {'ANSIBLE_ROLES_PATH': roles_path}
+    }
+    general_validation_runner = ansible_runner.run(**parameters)
+    # Remove the generated playbook
+    remove_file(general_playbook)
+    # If the general validations have failed, then abort the execution finishing with an error. Else, continue.
+    if general_validation_runner.status == 'failed':
+        # Collect inventory_hostnames with errors
+        hosts_with_errors = [key for key in general_validation_runner.stats['failures']]
+        # Collect list of errors
+        errors = []
+        errors.extend([general_validation_runner.get_fact_cache(host)['phase_results'] for host in hosts_with_errors])
+        errors = ''.join(errors)
+        # Raise the exception with errors details
+        raise Exception(f"The general validations have failed. Please check that the environments meet the expected "
+                        f"requirements. Result:\n{errors}")
+    # -------------------------------------------------- End of Step 3 -------------------------------------------------
+
+
+@pytest.fixture(scope='module', autouse=True)
+def run_specific_validations(request):
+    """Fixture with module scope to validate the environment of an specific tests with specific validation tasks.
+
+    Execute a test-specific playbook (if any). This will run one validation playbook for each test module.
+
+    Args:
+        request (fixture):  Gives access to the requesting test context.
+    """
+    roles_path = request.config.getoption('--roles-path')
+    inventory_path = request.config.getoption('--inventory_path')
+    test_suite_path = os.path.dirname(request.fspath)
+    test_suite_name = test_suite_path.split('/')[-1:][0]
+    target_hosts, target_distros = get_target_hosts_and_distros(test_suite_name)
+    validation_playbook = os.path.join(test_suite_path, 'data', 'playbooks', 'validation.yaml')
+
+    # Run test-specific validation playbook (if any)
+    if os.path.exists(validation_playbook):
+        parameters = {
+            'playbook': validation_playbook, 'inventory': inventory_path,
+            'envvars': {'ANSIBLE_ROLES_PATH': roles_path},
+            'extravars': {'target_hosts': ','.join(target_hosts), 'distros': target_distros}
+        }
+        validation_runner = ansible_runner.run(**parameters)
+
+        # If the validation phase has failed, then abort the execution finishing with an error. Else, continue.
+        if validation_runner.status == 'failed':
+            raise Exception(f"The validation phase of {test_suite_name} has failed. Please check that the "
+                            'environments meet the expected requirements.')
 
 
 @pytest.fixture(scope='function')
@@ -152,3 +283,12 @@ def pytest_addoption(parser):
         type=str,
         help='Inventory path',
     )
+
+    parser.addoption(
+        '--roles-path',
+        action='store',
+        metavar='ROLES_PATH',
+        default=os.path.join(suite_path, 'data', 'ansible_roles'),
+        type=str,
+        help='Ansible roles path.',
+    )

tests/end_to_end/data/ansible_roles/host_checker/tasks/check_controller_indexer.yaml

@@ -0,0 +1,22 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+
+- name: Get Wazuh installation
+  include_role:
+    name: service_controller
+    tasks_from: get_installation_type
+
+- name: Test connection with Wazuh Indexer
+  shell: nc -v -4 {{ hostvars[inventory_hostname]['ansible_host'] }} 9200
+  timeout: 3
+  ignore_errors: true
+  register: test_result
+  delegate_to: localhost
+  when: (os == 'Linux' and 'server' in wazuh_info.stdout)
+
+- name: Check the connection between Controller node and Wazuh Indexer
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}Ansible Controller node cannot connect correctly with Wazuh Indexer.\n"
+  when: (test_result is failed and test_result.stderr is defined and 'refused' in test_result.stderr)

tests/end_to_end/data/ansible_roles/host_checker/tasks/check_filebeat_indexer.yaml

@@ -0,0 +1,21 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+
+- name: Get Wazuh installation
+  include_role:
+    name: service_controller
+    tasks_from: get_installation_type
+
+- name: Run filebeat test
+  become: true
+  shell: filebeat test output
+  register: test_result
+  ignore_errors: true
+  when: (os == 'Linux' and 'server' in wazuh_info.stdout)
+
+- name: Check the connection between Filebeat and Wazuh Indexer
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}Filebeat cannot connect correctly with Wazuh Indexer.\n"
+  when: (os == 'Linux' and 'server' in wazuh_info.stdout and 'ERROR' in test_result.stdout)

tests/end_to_end/data/ansible_roles/host_checker/tasks/check_os.yaml

@@ -0,0 +1,16 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+# (String) supported_distros: List of ditros supported by the current test
+
+- name: Check OS (Linux)
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}The {{ ansible_distribution }} distro isn't supported for the selected tests currently.\n"
+  when: (ansible_distribution not in supported_distros and os == "Linux")
+
+- name: Check OS (Windows)
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}The {{ os }} OS isn't supported for the selected tests currently.\n"
+  when: (os == "Windows" and os not in supported_distros)

tests/end_to_end/data/ansible_roles/host_checker/tasks/check_python.yaml

@@ -0,0 +1,20 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+
+- name: Check default Python version (Linux)
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}Python version is less than 3. Current version: {{ ansible_python_version }}\n"
+  when: (os == "Linux" and ansible_python['version']['major'] < 3)
+
+- name: Get Python version (Windows)
+  win_shell: python -V
+  register: version
+  when: os == 'Windows'
+
+- name: Check default Python version (Windows)
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}Python version is less than 3. Current version: {{ version.stdout }}\n"
+  when: (os == "Windows" and version.stdout.split(" ")[1].split(".")[0] | int < 3)

tests/end_to_end/data/ansible_roles/host_checker/tasks/check_wazuh_components.yaml

@@ -0,0 +1,32 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+
+- name: Get Wazuh installation
+  include_role:
+    name: service_controller
+    tasks_from: get_installation_type
+
+- name: Populate services facts
+  service_facts:
+  when: os == 'Linux'
+
+- name: Check the status of Wazuh components (Manager)
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}{{ ansible_facts.services[item].name }} is not running.\n"
+  when: (os == 'Linux' and 'server' in wazuh_info.stdout and ansible_facts.services[item].state != 'running')
+  with_items:
+    - wazuh-manager.service
+    - wazuh-indexer.service
+    - filebeat.service
+
+- set_fact:
+    service: wazuh-agent.service
+  when: (os == 'Linux' and 'agent' in wazuh_info.stdout)
+
+- name: Check the status of Wazuh Agent
+  set_fact:
+    check_result: 'true'
+    errors: "{{ errors }}{{ ansible_facts.services[service].name }} is not running.\n"
+  when: (os == 'Linux' and 'agent' in wazuh_info.stdout and ansible_facts.services[service].state != 'running')

tests/end_to_end/data/ansible_roles/host_checker/tasks/main.yaml

@@ -0,0 +1,34 @@
+# -------- Task to identify whether the validation step fails or not. --------
+- name: Set flag and informative variable
+  set_fact:
+    check_result: 'false'
+    errors: ''
+# ----------------------------------------------------------------------------
+
+# -------- Checks ------------------------------------------------------------
+- name: Check Python
+  import_tasks: check_python.yaml
+
+- name: Check OS
+  import_tasks: check_os.yaml
+
+- name: Check the status of Wazuh components
+  import_tasks: check_wazuh_components.yaml
+
+- name: Check the connection between Filebeat and Wazuh Indexer
+  import_tasks: check_filebeat_indexer.yaml
+
+- name: Check the connection between Controller node and Wazuh Indexer
+  import_tasks: check_controller_indexer.yaml
+# ----------------------------------------------------------------------------
+
+# -------- Task to identify whether the validation step fails or not. --------
+- set_fact:
+    phase_results: "{{ errors }}"
+    cacheable: true
+
+- name: Verify if any check have failed
+  fail:
+    msg: "Some validations were fail:\n'{{ errors }}'"
+  when: (check_result == 'true' or errors != '')
+# ----------------------------------------------------------------------------

tests/end_to_end/data/ansible_roles/service_controller/tasks/get_installation_type.yaml

@@ -0,0 +1,9 @@
+# REQUIRED VARIABLES
+# -------------------
+# (String) os: Target operating system
+
+- name: Get installation type
+  become: true
+  shell: /var/ossec/bin/wazuh-control info
+  register: wazuh_info
+  when: os == 'Linux'