From bddc22854d7b897127f080b10b48c162d2f9f49b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 9 Sep 2024 13:48:25 +0200 Subject: [PATCH 01/19] Added provision and deletion of instances --- ...est_installation_assistant_distributed.yml | 205 ++++++++++++++++-- 1 file changed, 192 insertions(+), 13 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index b2d91aa..06b169b 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -1,5 +1,5 @@ -run-name: (Distributed) Test installation assistant - Launched by @${{ github.actor }} -name: (Distributed) Test installation assistant +run-name: (Distributed) Test installation assistant - ${{ github.run_id }} - ${{ inputs.SYSTEMS }} - Launched by @${{ github.actor }} +name: (Distributed) Test installation assistant on: pull_request: @@ -21,14 +21,28 @@ on: - staging - pre-release AUTOMATION_REFERENCE: - description: 'wazuh-automation reference' + description: 'Branch or tag of the wazuh-automation repository' required: true - default: 'v4.10.0' - DEBUG: - description: 'Debug mode' + default: '4.10.0' + ASSISTANT_REFERENCE: + description: 'Branch or tag of the wazuh-installation-assistant repository' required: true - default: false - type: boolean + default: '4.10.0' + SYSTEMS: + description: 'Operating Systems (list of comma-separated quoted strings enclosed in square brackets)' + required: true + default: '["CentOS_8", "AmazonLinux_2","Ubuntu_22", "RHEL8"]' + type: string + VERBOSITY: + description: 'Verbosity level on playbooks execution' + required: true + default: '-v' + type: choice + options: + - -v + - -vv + - -vvv + - -vvvv DESTROY: description: 'Destroy instances after run' required: true @@ -36,12 +50,177 @@ on: type: boolean env: - LABEL: ubuntu-latest + COMPOSITE_NAME: "linux-SUBNAME-amd64" + SESSION_NAME: "Installation-Assistant-Test" + REGION: "us-east-1" + TMP_PATH: "/tmp/test" + LOGS_PATH: "${{ github.workspace }}/assistant_logs" + PKG_REPOSITORY: "${{ inputs.REPOSITORY }}" + TEST_NAME: "test_assistant_distributed" + REPOSITORY_URL: "${{ github.server_url }}/${{ github.repository }}.git" + ALLOCATOR_PATH: "/tmp/allocator_instance" + INSTANCE_NAMES: "instance_1 instance_2 instance_3" + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout jobs: - initialize-environment: - runs-on: $LABEL + run-test: + runs-on: ubuntu-latest + strategy: + fail-fast: false # If a job fails, the rest of jobs will not be canceled + matrix: + system: ${{ fromJson(inputs.SYSTEMS) }} steps: - - name: Set up Git - uses: actions/checkout@v3 + - name: Checkout code + uses: actions/checkout@v4 + + - name: View parameters + run: echo "${{ toJson(inputs) }}" + + - name: Set COMPOSITE_NAME variable + run: | + case "${{ matrix.system }}" in + "CentOS_7") + SUBNAME="centos-7" + ;; + "CentOS_8") + SUBNAME="centos-8" + ;; + "AmazonLinux_2") + SUBNAME="amazon-2" + ;; + "Ubuntu_16") + SUBNAME="ubuntu-16.04" + ;; + "Ubuntu_18") + SUBNAME="ubuntu-18.04" + ;; + "Ubuntu_20") + SUBNAME="ubuntu-20.04" + ;; + "Ubuntu_22") + SUBNAME="ubuntu-22.04" + ;; + "RHEL7") + SUBNAME="redhat-7" + ;; + "RHEL8") + SUBNAME="redhat-8" + ;; + *) + echo "Invalid SYSTEM selection" >&2 + exit 1 + ;; + esac + COMPOSITE_NAME="${COMPOSITE_NAME/SUBNAME/$SUBNAME}" + echo "COMPOSITE_NAME=$COMPOSITE_NAME" >> $GITHUB_ENV + + - name: Install Ansible + run: sudo apt-get update && sudo apt install -y python3 && python3 -m pip install --user ansible-core==2.16 + + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_IAM_ROLE }} + role-session-name: ${{ env.SESSION_NAME }} + aws-region: ${{ env.REGION }} + + - name: Checkout wazuh/wazuh-automation repository + uses: actions/checkout@v4 + with: + repository: wazuh/wazuh-automation + ref: ${{ inputs.AUTOMATION_REFERENCE }} + token: ${{ secrets.GH_CLONE_TOKEN }} + path: wazuh-automation + + - name: Install and set allocator requirements + run: pip3 install -r wazuh-automation/deployability/deps/requirements.txt + + - name: Allocate instances and create inventory + id: allocator_instance + run: | + instance_names=($INSTANCE_NAMES) + inventory_file="$ALLOCATOR_PATH/inventory" + inventory_indexers="$ALLOCATOR_PATH/inventory_indexers" + inventory_managers="$ALLOCATOR_PATH/inventory_managers" + inventory_dashboards="$ALLOCATOR_PATH/inventory_dashboards" + inventory_common="$ALLOCATOR_PATH/inventory_common" + inventory_file="$ALLOCATOR_PATH/inventory" + + mkdir -p $ALLOCATOR_PATH + echo "[indexers]" > $inventory_indexers + echo "[managers]" > $inventory_managers + echo "[dashboards]" > $inventory_dashboards + echo "[all:vars]" > $inventory_common + + for i in ${!instance_names[@]}; do + instance_name=${instance_names[$i]} + # Provision instance in parallel + ( + python3 wazuh-automation/deployability/modules/allocation/main.py \ + --action create --provider aws --size large \ + --composite-name ${{ env.COMPOSITE_NAME }} \ + --working-dir $ALLOCATOR_PATH --track-output $ALLOCATOR_PATH/track_${instance_name}.yml \ + --inventory-output $ALLOCATOR_PATH/inventory_${instance_name}.yml \ + --instance-name gha_${{ github.run_id }}_${{ env.TEST_NAME }}_${instance_name} --label-team devops --label-termination-date 1d + + # Modify inventory for Ansible + sed 's/: */=/g' $ALLOCATOR_PATH/inventory_${instance_name}.yml > $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + sed -i 's/-o StrictHostKeyChecking=no/\"-o StrictHostKeyChecking=no\"/g' $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + source $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml + + # Add instance to corresponding group + if [[ $i -eq 0 ]]; then + echo "indexer1 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "master ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=master" >> $inventory_managers + + echo "ansible_user=$ansible_user" >> $inventory_common + echo "ansible_port=$ansible_port" >> $inventory_common + echo "ansible_ssh_common_args='$ansible_ssh_common_args'" >> $inventory_common + elif [[ $i -eq 1 ]]; then + echo "indexer2 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "worker1 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + else + echo "indexer3 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "worker2 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + echo "dashboard ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_dashboards + fi + ) & + done + + # Wait for all provisioning tasks to complete + wait + + # Combine the temporary inventories into one + cat $inventory_indexers > $inventory_file + cat $inventory_managers >> $inventory_file + cat $inventory_dashboards >> $inventory_file + cat $inventory_common >> $inventory_file + + + cat $inventory_file + + - name: Delete allocated VMs + if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true + run: | + instance_names=($INSTANCE_NAMES) + + for i in ${!instance_names[@]}; do + instance_name=${instance_names[$i]} + track_file="$ALLOCATOR_PATH/track_${instance_name}.yml" + + echo "Deleting instance: $instance_name using track file $track_file" + + ( + # Delete instance + python3 wazuh-automation/deployability/modules/allocation/main.py \ + --action delete --provider aws --track-output $track_file + ) & + done + + # Wait for all deletion tasks to complete + wait + From 758dda9ffd22ab3e74e31b36186260b0d026a132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 9 Sep 2024 16:41:00 +0200 Subject: [PATCH 02/19] Added private IP capture --- ...est_installation_assistant_distributed.yml | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 06b169b..36f57bd 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -167,26 +167,31 @@ jobs: --inventory-output $ALLOCATOR_PATH/inventory_${instance_name}.yml \ --instance-name gha_${{ github.run_id }}_${{ env.TEST_NAME }}_${instance_name} --label-team devops --label-termination-date 1d - # Modify inventory for Ansible + instance_id=$(grep '^identifier' $ALLOCATOR_PATH/track_${instance_name}.yml | awk '{print $2}') + private_ip=$(aws ec2 describe-instances \ + --instance-ids $instance_id \ + --query 'Reservations[*].Instances[*].PrivateIpAddress' \ + --output text) + sed 's/: */=/g' $ALLOCATOR_PATH/inventory_${instance_name}.yml > $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml sed -i 's/-o StrictHostKeyChecking=no/\"-o StrictHostKeyChecking=no\"/g' $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml source $ALLOCATOR_PATH/inventory_mod_${instance_name}.yml # Add instance to corresponding group if [[ $i -eq 0 ]]; then - echo "indexer1 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "master ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=master" >> $inventory_managers + echo "indexer1 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "master ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=master" >> $inventory_managers echo "ansible_user=$ansible_user" >> $inventory_common echo "ansible_port=$ansible_port" >> $inventory_common echo "ansible_ssh_common_args='$ansible_ssh_common_args'" >> $inventory_common elif [[ $i -eq 1 ]]; then - echo "indexer2 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "worker1 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + echo "indexer2 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "worker1 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers else - echo "indexer3 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "worker2 ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers - echo "dashboard ansible_host=$ansible_host ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_dashboards + echo "indexer3 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers + echo "worker2 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + echo "dashboard ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_dashboards fi ) & done @@ -199,7 +204,6 @@ jobs: cat $inventory_managers >> $inventory_file cat $inventory_dashboards >> $inventory_file cat $inventory_common >> $inventory_file - cat $inventory_file From e6fe051d717b7127ce8d31e8a3930cadb6e20ef9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 10:45:38 +0200 Subject: [PATCH 03/19] Added certificates generation logic --- ...est_installation_assistant_distributed.yml | 14 ++++- .../distributed_generate_certificates.yml | 56 +++++++++++++++++++ .github/workflows/templates/config.j2 | 18 ++++++ 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ansible-playbooks/distributed_generate_certificates.yml create mode 100644 .github/workflows/templates/config.j2 diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 36f57bd..3086eac 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -204,8 +204,18 @@ jobs: cat $inventory_managers >> $inventory_file cat $inventory_dashboards >> $inventory_file cat $inventory_common >> $inventory_file - - cat $inventory_file + + - name: Execute certificates generation playbook + run: | + TEST_NAME="${{ env.TEST_NAME }}_certificates" + RESOURCES_PATH="${{ github.workspace }}" + ansible-playbook .github/workflows/ansible-playbooks/distributed_generate_certificates.yml \ + -i $ALLOCATOR_PATH/inventory \ + -e "resources_path=$RESOURCES_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + -e "pkg_repository=$PKG_REPOSITORY" \ + "${{ inputs.VERBOSITY }}" - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true diff --git a/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml b/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml new file mode 100644 index 0000000..8d00600 --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml @@ -0,0 +1,56 @@ +--- +- hosts: localhost + become: true + + vars: + test_name: "{{ test_name }}" + test_dir: "/{{ test_name }}" + logs_path: "{{ log_folder }}" + resources_path: "{{ resources_path }}" + pkg_repository: "{{ pkg_repository }}" + + tasks: + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}.log" + state: touch + + - name: Create certificates + block: + - name: Rendering wazuh-config file + template: + src: "{{ resources_path }}/.github/workflows/templates/config.j2" + dest: "{{ resources_path }}/config.yml" + mode: "664" + force: yes + + - name: Creating wazuh-install.sh script + shell: "bash {{ resources_path }}/builder.sh -i -d {{ pkg_repository }}" + + - name: Creating Certificates + shell: "bash {{ resources_path }}/wazuh-install.sh -g -v" + register: certificates_install + + - name: Give read permissions to wazuh-install-files.tar + shell: "chmod +r {{ resources_path }}/wazuh-install-files.tar" + + always: + - name: Save output certificate build and creation + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}.log" + block: | + {{ certificates_install.stderr }} + -------------------------------- + {{ certificates_install.stdout }} + + - name: Fetch log + fetch: + src: "{{ test_dir }}/{{ test_name }}.log" + dest: "{{ logs_path }}/" + flat: yes diff --git a/.github/workflows/templates/config.j2 b/.github/workflows/templates/config.j2 new file mode 100644 index 0000000..44af40b --- /dev/null +++ b/.github/workflows/templates/config.j2 @@ -0,0 +1,18 @@ +nodes: + # Wazuh indexer nodes + indexer: +{% for indexer in groups['indexers'] %} + - name: {{ hostvars[indexer]['inventory_hostname'] }} + ip: "{{ hostvars[indexer]['private_ip'] }}" +{% endfor %} + server: +{% for manager in groups['managers'] %} + - name: {{ hostvars[manager]['inventory_hostname'] }} + ip: "{{ hostvars[manager]['private_ip'] }}" + node_type: "{{ hostvars[manager]['manager_type'] }}" +{% endfor %} + dashboard: +{% for dashboard in groups['dashboards'] %} + - name: {{ hostvars[dashboard]['inventory_hostname'] }} + ip: "{{ hostvars[dashboard]['private_ip'] }}" +{% endfor %} From 5fbb84fcf5a80a28400959dcb70c7b9d238f6da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 10:56:14 +0200 Subject: [PATCH 04/19] Added certificates generation logic --- ...Test_installation_assistant_distributed.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 3086eac..dc96551 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -205,6 +205,24 @@ jobs: cat $inventory_dashboards >> $inventory_file cat $inventory_common >> $inventory_file + - name: Execute provision playbook + run: | + INSTALL_DEPS=true + INSTALL_PYTHON=true + INSTALL_PIP_DEPS=true + + ansible-playbook .github/workflows/ansible-playbooks/provision.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l indexers \ + -e "repository=$REPOSITORY_URL" \ + -e "reference=${{ inputs.ASSISTANT_REFERENCE }}" \ + -e "tmp_path=$TMP_PATH" \ + -e "pkg_repository=$PKG_REPOSITORY" \ + -e "install_deps=$INSTALL_DEPS" \ + -e "install_python=$INSTALL_PYTHON" \ + -e "install_pip_deps=$INSTALL_PIP_DEPS" \ + "${{ inputs.VERBOSITY }}" + - name: Execute certificates generation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_certificates" From ca9051f0801ee72d6c33d705bd3a4ac854627ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 11:05:53 +0200 Subject: [PATCH 05/19] Added certificates copy playbook execution --- .../Test_installation_assistant_distributed.yml | 12 +++++++++++- .../distributed_copy_certificates.yml | 12 ++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ansible-playbooks/distributed_copy_certificates.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index dc96551..8a7e64a 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -54,6 +54,7 @@ env: SESSION_NAME: "Installation-Assistant-Test" REGION: "us-east-1" TMP_PATH: "/tmp/test" + RESOURCES_PATH: "${{ github.workspace }}" LOGS_PATH: "${{ github.workspace }}/assistant_logs" PKG_REPOSITORY: "${{ inputs.REPOSITORY }}" TEST_NAME: "test_assistant_distributed" @@ -226,7 +227,6 @@ jobs: - name: Execute certificates generation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_certificates" - RESOURCES_PATH="${{ github.workspace }}" ansible-playbook .github/workflows/ansible-playbooks/distributed_generate_certificates.yml \ -i $ALLOCATOR_PATH/inventory \ -e "resources_path=$RESOURCES_PATH" \ @@ -234,6 +234,16 @@ jobs: -e "test_name=$TEST_NAME" \ -e "pkg_repository=$PKG_REPOSITORY" \ "${{ inputs.VERBOSITY }}" + + - name: Copy certificates to nodes + run: | + TEST_NAME="${{ env.TEST_NAME }}_copy_certificates" + ansible-playbook .github/workflows/ansible-playbooks/distributed_copy_certificates.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l indexers \ + -e "tmp_path=$TMP_PATH" \ + -e "resources_path=$RESOURCES_PATH" \ + "${{ inputs.VERBOSITY }}" - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true diff --git a/.github/workflows/ansible-playbooks/distributed_copy_certificates.yml b/.github/workflows/ansible-playbooks/distributed_copy_certificates.yml new file mode 100644 index 0000000..083b8f4 --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_copy_certificates.yml @@ -0,0 +1,12 @@ +--- + - hosts: all + gather_facts: false + tasks: + - name: Copying the wazuh-install-files.tar to the instances + copy: + src: "{{ resources_path }}/wazuh-install-files.tar" + dest: "{{ tmp_path }}/" + force: yes + remote_src: no + become: yes + become_user: root From 48bb2b86eb29ac381e754a2c250dd7ebe1e0f9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 11:20:14 +0200 Subject: [PATCH 06/19] Added indexer install playbook execution --- ...est_installation_assistant_distributed.yml | 11 +++++ .../distributed_install_indexer.yml | 44 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 .github/workflows/ansible-playbooks/distributed_install_indexer.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 8a7e64a..891f40b 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -245,6 +245,17 @@ jobs: -e "resources_path=$RESOURCES_PATH" \ "${{ inputs.VERBOSITY }}" + - name: Execute indexer installation playbook + run: | + TEST_NAME="${{ env.TEST_NAME }}_indexer" + ansible-playbook .github/workflows/ansible-playbooks/distributed_install_indexer.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l indexers \ + -e "tmp_path=$TMP_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + "${{ inputs.VERBOSITY }}" + - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true run: | diff --git a/.github/workflows/ansible-playbooks/distributed_install_indexer.yml b/.github/workflows/ansible-playbooks/distributed_install_indexer.yml new file mode 100644 index 0000000..7f41bd7 --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_install_indexer.yml @@ -0,0 +1,44 @@ +--- + - hosts: all + gather_facts: false + become: true + + vars: + test_name: "{{ test_name }}" + test_dir: "/{{ test_name }}" + logs_path: "{{ logs_path }}" + + tasks: + + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + state: touch + + - name: Install Wazuh indexer + block: + - name: Install Wazuh indexer + command: "bash {{ tmp_path }}/wazuh-install.sh -wi {{ inventory_hostname }} -v" + register: indexer + + always: + + - name: Save output Wazuh indexer + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + block: | + {{ indexer.stderr }} + -------------------------------- + {{ indexer.stdout }} + + - name: Fetch log + fetch: + src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + dest: "{{ logs_path }}/" + flat: yes From ddbadbca4e20b5b6da28a8a425716eef9be738e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 11:34:12 +0200 Subject: [PATCH 07/19] Added server playbook and task in distributed workflow --- ...est_installation_assistant_distributed.yml | 11 +++++ .../distributed_install_wazuh.yml | 44 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 .github/workflows/ansible-playbooks/distributed_install_wazuh.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 891f40b..b1ca31f 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -255,6 +255,17 @@ jobs: -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" + + - name: Execute server installation playbook + run: | + TEST_NAME="${{ env.TEST_NAME }}_server" + ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l managers \ + -e "tmp_path=$TMP_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + "${{ inputs.VERBOSITY }}" - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true diff --git a/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml new file mode 100644 index 0000000..a116940 --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml @@ -0,0 +1,44 @@ +--- +- hosts: all + gather_facts: false + become: true + + vars: + test_name: "{{ test_name }}" + tmp_path: "{{ tmp_path }}" + test_dir: "/{{ test_name }}" + logs_path: "{{ logs_path }}" + + tasks: + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + state: touch + + - name: Install Wazuh + block: + + - name: Install Wazuh server + command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" + register: wazuh + + always: + + - name: Save output Wazuh + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + block: | + {{ wazuh.stderr }} + -------------------------------- + {{ wazuh.stdout }} + - name: Fetch log + fetch: + src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + dest: "{{ logs_path }}/" + flat: yes \ No newline at end of file From d70f8260e34738bdc6371a029302cf4f7e42e6f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 11:35:18 +0200 Subject: [PATCH 08/19] Added dashboard install playbook execution --- ...est_installation_assistant_distributed.yml | 11 +++++ .../distributed_install_dashboard.yml | 46 +++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 .github/workflows/ansible-playbooks/distributed_install_dashboard.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index b1ca31f..56ddcae 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -266,6 +266,17 @@ jobs: -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" + + - name: Execute dashboard installation playbook + run: | + TEST_NAME="${{ env.TEST_NAME }}_dashboard" + ansible-playbook .github/workflows/ansible-playbooks/distributed_install_dashboard.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l dashboards \ + -e "tmp_path=$TMP_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + "${{ inputs.VERBOSITY }}" - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true diff --git a/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml b/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml new file mode 100644 index 0000000..691e17a --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml @@ -0,0 +1,46 @@ +--- +- hosts: all + gather_facts: false + become: true + + vars: + test_name: "{{ test_name }}" + test_dir: "/{{ test_name }}" + tmp_path: "{{ tmp_path }}" + logs_path: "{{ logs_path }}" + + tasks: + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + state: touch + + - name: Install Wazuh dashboard + block: + + - name: Install Wazuh dashboard + command: "bash wazuh-install.sh -wd {{ inventory_hostname }} -v" + args: + chdir: "{{ tmp_path }}" + register: dashboard + + always: + + - name: Save output Wazuh dashboard + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + block: | + {{ dashboard.stderr }} + -------------------------------- + {{ dashboard.stdout }} + - name: Fetch log + fetch: + src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + dest: "{{ logs_path }}/" + flat: yes \ No newline at end of file From cc14906e99fd80f73d4ac0ef09d25f945e376e41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 11:54:38 +0200 Subject: [PATCH 09/19] Added indexer cluster start playbook execution --- ...est_installation_assistant_distributed.yml | 11 +++++ .../distributed_start_indexer_cluster.yml | 49 +++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 56ddcae..afff1c4 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -266,6 +266,17 @@ jobs: -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" + + - name: Execute indexer cluster start playbook + run: | + TEST_NAME="${{ env.TEST_NAME }}_start_cluster" + ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l indexers \ + -e "tmp_path=$TMP_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + "${{ inputs.VERBOSITY }}" - name: Execute dashboard installation playbook run: | diff --git a/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml new file mode 100644 index 0000000..12506bd --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml @@ -0,0 +1,49 @@ +--- +- hosts: all + gather_facts: false + become: true + + vars: + test_name: "{{ test_name }}" + test_dir: "/{{ test_name }}" + tmp_path: "{{ tmp_path }}" + logs_path: "{{ logs_path }}" + + tasks: + - name: Start Wazuh indexer cluster in just one node + block: + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + state: touch + + - name: Start cluster + block: + + - name: Start Wazuh indexer cluster + command: "bash {{ tmp_path }}/wazuh-install.sh -s -v" + register: cluster + + always: + + - name: Save output start cluster + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + block: | + {{ cluster.stderr }} + -------------------------------- + {{ cluster.stdout }} + + - name: Fetch log + fetch: + src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + dest: "{{ logs_path }}/" + flat: yes + when: inventory_hostname == ansible_play_hosts[0] + From b0ff7f4192333e2dbdb32d99f21ebb4320ba6368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 10 Sep 2024 12:31:32 +0200 Subject: [PATCH 10/19] Changed indexer cluster playbook execution order --- ...Test_installation_assistant_distributed.yml | 18 ++++++++++-------- .../distributed_start_indexer_cluster.yml | 1 - 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index afff1c4..e7c9c5e 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -256,23 +256,25 @@ jobs: -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" - - name: Execute server installation playbook + - name: Execute indexer cluster start playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_server" - ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ + TEST_NAME="${{ env.TEST_NAME }}_start_cluster" + INDEXER_ADMIN_PASSWORD="admin" + ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ -i $ALLOCATOR_PATH/inventory \ - -l managers \ + -l indexers \ -e "tmp_path=$TMP_PATH" \ -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ + -e "indexer_admin_password=$INDEXER_ADMIN_PASSWORD" \ "${{ inputs.VERBOSITY }}" - - name: Execute indexer cluster start playbook + - name: Execute server installation playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_start_cluster" - ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ + TEST_NAME="${{ env.TEST_NAME }}_server" + ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ -i $ALLOCATOR_PATH/inventory \ - -l indexers \ + -l managers \ -e "tmp_path=$TMP_PATH" \ -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ diff --git a/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml index 12506bd..4c94f8a 100644 --- a/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml +++ b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml @@ -24,7 +24,6 @@ - name: Start cluster block: - - name: Start Wazuh indexer cluster command: "bash {{ tmp_path }}/wazuh-install.sh -s -v" register: cluster From b29e4e5b36f9d27514045e51f566498a9b369ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Fri, 13 Sep 2024 10:47:02 +0200 Subject: [PATCH 11/19] Workers wait master node to be installed --- .../distributed_install_wazuh.yml | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml index a116940..26a5fae 100644 --- a/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml +++ b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml @@ -8,6 +8,10 @@ tmp_path: "{{ tmp_path }}" test_dir: "/{{ test_name }}" logs_path: "{{ logs_path }}" + master_ip: "{{ hostvars[groups['managers'] | select('match', 'master') | first]['private_ip'] }}" + check_port: 55000 + retries: 10 + delay: 30 tasks: - name: Create log directory @@ -20,16 +24,39 @@ dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" state: touch - - name: Install Wazuh + - name: Install Wazuh server on master block: - - - name: Install Wazuh server + - name: Install Wazuh server (Master) command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" register: wazuh + + - name: Save Wazuh installation log (Master) + blockinfile: + marker: "" + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + block: | + {{ wazuh.stderr }} + -------------------------------- + {{ wazuh.stdout }} + when: hostvars[inventory_hostname].manager_type == 'master' - always: + - name: Install Wazuh server on worker nodes + block: + - name: Wait for Wazuh master to be ready on port {{ check_port }} + wait_for: + host: "{{ master_ip }}" + port: "{{ check_port }}" + delay: "{{ delay }}" + timeout: 300 + when: hostvars[inventory_hostname].manager_type == 'worker' + async: 500 + poll: 5 - - name: Save output Wazuh + - name: Install Wazuh server (Workers) + command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" + register: wazuh + + - name: Save Wazuh installation log (Workers) blockinfile: marker: "" path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" @@ -37,8 +64,6 @@ {{ wazuh.stderr }} -------------------------------- {{ wazuh.stdout }} - - name: Fetch log - fetch: - src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - dest: "{{ logs_path }}/" - flat: yes \ No newline at end of file + when: hostvars[inventory_hostname].manager_type == 'worker' + + From 7244372ad27bfc365cd4d2045f92057e9ebf6e9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Fri, 13 Sep 2024 15:36:34 +0200 Subject: [PATCH 12/19] Added distributed test playbook execution --- ...est_installation_assistant_distributed.yml | 17 ++- .../ansible-playbooks/distributed_tests.yml | 110 ++++++++++++++++++ 2 files changed, 124 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/ansible-playbooks/distributed_tests.yml diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index e7c9c5e..0ffc24a 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -181,17 +181,17 @@ jobs: # Add instance to corresponding group if [[ $i -eq 0 ]]; then echo "indexer1 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "master ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=master" >> $inventory_managers + echo "master ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=master instance_type=indexer_manager" >> $inventory_managers echo "ansible_user=$ansible_user" >> $inventory_common echo "ansible_port=$ansible_port" >> $inventory_common echo "ansible_ssh_common_args='$ansible_ssh_common_args'" >> $inventory_common elif [[ $i -eq 1 ]]; then echo "indexer2 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "worker1 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + echo "worker1 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker instance_type=indexer_manager" >> $inventory_managers else echo "indexer3 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_indexers - echo "worker2 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker" >> $inventory_managers + echo "worker2 ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file manager_type=worker instance_type=indexer_manager_dashboard" >> $inventory_managers echo "dashboard ansible_host=$ansible_host private_ip=$private_ip ansible_ssh_private_key_file=$ansible_ssh_private_key_file" >> $inventory_dashboards fi ) & @@ -290,6 +290,17 @@ jobs: -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" + + - name: Execute Python test playbook + run: | + TEST_NAME="test_installation_assistant" + ansible-playbook .github/workflows/ansible-playbooks/distributed_tests.yml \ + -i $ALLOCATOR_PATH/inventory \ + -l managers \ + -e "tmp_path=$TMP_PATH" \ + -e "logs_path=$LOGS_PATH" \ + -e "test_name=$TEST_NAME" \ + "${{ inputs.VERBOSITY }}" - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true diff --git a/.github/workflows/ansible-playbooks/distributed_tests.yml b/.github/workflows/ansible-playbooks/distributed_tests.yml new file mode 100644 index 0000000..b3fbf54 --- /dev/null +++ b/.github/workflows/ansible-playbooks/distributed_tests.yml @@ -0,0 +1,110 @@ +--- +- hosts: all + gather_facts: false + become: true + + vars: + script_path: "{{ tmp_path }}/tests/install" + test_name: "{{ test_name }}" + script_name: "{{ test_name }}.py" + test_dir: "/{{ test_name }}" + logs_path: "{{ logs_path }}" + + tasks: + + - name: Create log directory + file: + path: "{{ test_dir }}" + state: directory + + - name: Create log file + file: + dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + state: touch + + - name: Test distributed master + block: + + - name: Execute tests Master + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_cluster\"" + args: + chdir: "{{ script_path }}" + register: test_results_master + when: + - hostvars[inventory_hostname].manager_type == 'master' + + - name: Execute tests Workers + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_worker\"" + args: + chdir: "{{ script_path }}" + register: test_results_worker + when: + - hostvars[inventory_hostname].manager_type == 'worker' + + - name: Execute tests indexer_manager + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster\"" + args: + chdir: "{{ script_path }}" + register: test_results_indexer_manager + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager' + + - name: Execute tests indexer_manager_dashboard + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster or dashboard\"" + args: + chdir: "{{ script_path }}" + register: test_results_indexer_manager_dashboard + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager_dashboard' + + always: + + - name: Save output Master + blockinfile: + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + marker: "" + block: | + {{ test_results_master.stderr }} + -------------------------------- + {{ test_results_master.stdout }} + when: + - hostvars[inventory_hostname].manager_type == 'master' + + - name: Save output Worker + blockinfile: + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + marker: "" + block: | + {{ test_results_worker.stderr }} + -------------------------------- + {{ test_results_worker.stdout }} + when: + - hostvars[inventory_hostname].manager_type == 'worker' + + - name: Save output indexer_manager + blockinfile: + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + marker: "" + block: | + {{ test_results_indexer_manager.stderr }} + -------------------------------- + {{ test_results_indexer_manager.stdout }} + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager' + + - name: Save output indexer_manager_dashboard + blockinfile: + path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + marker: "" + block: | + {{ test_results_indexer_manager_dashboard.stderr }} + -------------------------------- + {{ test_results_indexer_manager_dashboard.stdout }} + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager_dashboard' + + - name: Fetch logs + fetch: + src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" + dest: "{{ logs_path }}/" + flat: yes \ No newline at end of file From 008e2c7188e1dfa50b69a0cc5f55ddcb0126741f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Fri, 13 Sep 2024 16:23:38 +0200 Subject: [PATCH 13/19] Improving the playbooks output --- ...est_installation_assistant_distributed.yml | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 0ffc24a..0781347 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -54,6 +54,7 @@ env: SESSION_NAME: "Installation-Assistant-Test" REGION: "us-east-1" TMP_PATH: "/tmp/test" + ANSIBLE_CALLBACK: "yaml" RESOURCES_PATH: "${{ github.workspace }}" LOGS_PATH: "${{ github.workspace }}/assistant_logs" PKG_REPOSITORY: "${{ inputs.REPOSITORY }}" @@ -120,8 +121,8 @@ jobs: echo "COMPOSITE_NAME=$COMPOSITE_NAME" >> $GITHUB_ENV - name: Install Ansible - run: sudo apt-get update && sudo apt install -y python3 && python3 -m pip install --user ansible-core==2.16 - + run: sudo apt-get update && sudo apt install -y python3 && python3 -m pip install --user ansible-core==2.16 && pip install pyyaml && ansible-galaxy collection install community.general + - name: Set up AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -212,7 +213,7 @@ jobs: INSTALL_PYTHON=true INSTALL_PIP_DEPS=true - ansible-playbook .github/workflows/ansible-playbooks/provision.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/provision.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "repository=$REPOSITORY_URL" \ @@ -227,7 +228,7 @@ jobs: - name: Execute certificates generation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_certificates" - ansible-playbook .github/workflows/ansible-playbooks/distributed_generate_certificates.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_generate_certificates.yml \ -i $ALLOCATOR_PATH/inventory \ -e "resources_path=$RESOURCES_PATH" \ -e "logs_path=$LOGS_PATH" \ @@ -238,7 +239,7 @@ jobs: - name: Copy certificates to nodes run: | TEST_NAME="${{ env.TEST_NAME }}_copy_certificates" - ansible-playbook .github/workflows/ansible-playbooks/distributed_copy_certificates.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_copy_certificates.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "tmp_path=$TMP_PATH" \ @@ -248,7 +249,7 @@ jobs: - name: Execute indexer installation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_indexer" - ansible-playbook .github/workflows/ansible-playbooks/distributed_install_indexer.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_indexer.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "tmp_path=$TMP_PATH" \ @@ -260,7 +261,7 @@ jobs: run: | TEST_NAME="${{ env.TEST_NAME }}_start_cluster" INDEXER_ADMIN_PASSWORD="admin" - ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "tmp_path=$TMP_PATH" \ @@ -272,7 +273,7 @@ jobs: - name: Execute server installation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_server" - ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ -i $ALLOCATOR_PATH/inventory \ -l managers \ -e "tmp_path=$TMP_PATH" \ @@ -283,7 +284,7 @@ jobs: - name: Execute dashboard installation playbook run: | TEST_NAME="${{ env.TEST_NAME }}_dashboard" - ansible-playbook .github/workflows/ansible-playbooks/distributed_install_dashboard.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_dashboard.yml \ -i $ALLOCATOR_PATH/inventory \ -l dashboards \ -e "tmp_path=$TMP_PATH" \ @@ -294,7 +295,7 @@ jobs: - name: Execute Python test playbook run: | TEST_NAME="test_installation_assistant" - ansible-playbook .github/workflows/ansible-playbooks/distributed_tests.yml \ + ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_tests.yml \ -i $ALLOCATOR_PATH/inventory \ -l managers \ -e "tmp_path=$TMP_PATH" \ From c2d44bbfa2fca66ce114a8e99d7d09a33f813f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 16 Sep 2024 11:29:07 +0200 Subject: [PATCH 14/19] Added allocator info upload as artifact --- .../Test_installation_assistant_distributed.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 0781347..3487df8 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -302,6 +302,19 @@ jobs: -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" + + - name: Compress Allocator VM directory + id: compress_allocator_files + if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == false + run: | + zip -P "${{ secrets.ZIP_ARTIFACTS_PASSWORD }}" -r $ALLOCATOR_PATH.zip $ALLOCATOR_PATH + + - name: Upload Allocator VM directory as artifact + if: always() && steps.compress_allocator_files.outcome == 'success' && inputs.DESTROY == false + uses: actions/upload-artifact@v4 + with: + name: allocator-instance-${{ matrix.system }} + path: ${{ env.ALLOCATOR_PATH }}.zip - name: Delete allocated VMs if: always() && steps.allocator_instance.outcome == 'success' && inputs.DESTROY == true From 841f56881aec749b118ae1c9441b62e301e50c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 16 Sep 2024 16:28:23 +0200 Subject: [PATCH 15/19] Removed logs save logic --- ...est_installation_assistant_distributed.yml | 22 +-- .../distributed_generate_certificates.yml | 28 ---- .../distributed_install_dashboard.yml | 40 +----- .../distributed_install_indexer.yml | 37 +---- .../distributed_install_wazuh.yml | 41 +----- .../distributed_start_indexer_cluster.yml | 40 +----- .../ansible-playbooks/distributed_tests.yml | 131 +++++------------- 7 files changed, 46 insertions(+), 293 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 3487df8..6e0c23c 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -56,9 +56,8 @@ env: TMP_PATH: "/tmp/test" ANSIBLE_CALLBACK: "yaml" RESOURCES_PATH: "${{ github.workspace }}" - LOGS_PATH: "${{ github.workspace }}/assistant_logs" PKG_REPOSITORY: "${{ inputs.REPOSITORY }}" - TEST_NAME: "test_assistant_distributed" + TEST_NAME: "test_installation_assistant" REPOSITORY_URL: "${{ github.server_url }}/${{ github.repository }}.git" ALLOCATOR_PATH: "/tmp/allocator_instance" INSTANCE_NAMES: "instance_1 instance_2 instance_3" @@ -227,18 +226,14 @@ jobs: - name: Execute certificates generation playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_certificates" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_generate_certificates.yml \ -i $ALLOCATOR_PATH/inventory \ -e "resources_path=$RESOURCES_PATH" \ - -e "logs_path=$LOGS_PATH" \ - -e "test_name=$TEST_NAME" \ -e "pkg_repository=$PKG_REPOSITORY" \ "${{ inputs.VERBOSITY }}" - name: Copy certificates to nodes run: | - TEST_NAME="${{ env.TEST_NAME }}_copy_certificates" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_copy_certificates.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ @@ -248,58 +243,43 @@ jobs: - name: Execute indexer installation playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_indexer" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_indexer.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "tmp_path=$TMP_PATH" \ - -e "logs_path=$LOGS_PATH" \ - -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" - name: Execute indexer cluster start playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_start_cluster" INDEXER_ADMIN_PASSWORD="admin" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml \ -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "tmp_path=$TMP_PATH" \ - -e "logs_path=$LOGS_PATH" \ - -e "test_name=$TEST_NAME" \ - -e "indexer_admin_password=$INDEXER_ADMIN_PASSWORD" \ "${{ inputs.VERBOSITY }}" - name: Execute server installation playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_server" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_wazuh.yml \ -i $ALLOCATOR_PATH/inventory \ -l managers \ -e "tmp_path=$TMP_PATH" \ - -e "logs_path=$LOGS_PATH" \ - -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" - name: Execute dashboard installation playbook run: | - TEST_NAME="${{ env.TEST_NAME }}_dashboard" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_install_dashboard.yml \ -i $ALLOCATOR_PATH/inventory \ -l dashboards \ -e "tmp_path=$TMP_PATH" \ - -e "logs_path=$LOGS_PATH" \ - -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" - name: Execute Python test playbook run: | - TEST_NAME="test_installation_assistant" ANSIBLE_STDOUT_CALLBACK=$ANSIBLE_CALLBACK ansible-playbook .github/workflows/ansible-playbooks/distributed_tests.yml \ -i $ALLOCATOR_PATH/inventory \ -l managers \ -e "tmp_path=$TMP_PATH" \ - -e "logs_path=$LOGS_PATH" \ -e "test_name=$TEST_NAME" \ "${{ inputs.VERBOSITY }}" diff --git a/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml b/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml index 8d00600..0bfeb88 100644 --- a/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml +++ b/.github/workflows/ansible-playbooks/distributed_generate_certificates.yml @@ -3,23 +3,10 @@ become: true vars: - test_name: "{{ test_name }}" - test_dir: "/{{ test_name }}" - logs_path: "{{ log_folder }}" resources_path: "{{ resources_path }}" pkg_repository: "{{ pkg_repository }}" tasks: - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}.log" - state: touch - - name: Create certificates block: - name: Rendering wazuh-config file @@ -39,18 +26,3 @@ - name: Give read permissions to wazuh-install-files.tar shell: "chmod +r {{ resources_path }}/wazuh-install-files.tar" - always: - - name: Save output certificate build and creation - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}.log" - block: | - {{ certificates_install.stderr }} - -------------------------------- - {{ certificates_install.stdout }} - - - name: Fetch log - fetch: - src: "{{ test_dir }}/{{ test_name }}.log" - dest: "{{ logs_path }}/" - flat: yes diff --git a/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml b/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml index 691e17a..539bad4 100644 --- a/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml +++ b/.github/workflows/ansible-playbooks/distributed_install_dashboard.yml @@ -4,43 +4,11 @@ become: true vars: - test_name: "{{ test_name }}" - test_dir: "/{{ test_name }}" tmp_path: "{{ tmp_path }}" - logs_path: "{{ logs_path }}" tasks: - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - state: touch - - name: Install Wazuh dashboard - block: - - - name: Install Wazuh dashboard - command: "bash wazuh-install.sh -wd {{ inventory_hostname }} -v" - args: - chdir: "{{ tmp_path }}" - register: dashboard - - always: - - - name: Save output Wazuh dashboard - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - block: | - {{ dashboard.stderr }} - -------------------------------- - {{ dashboard.stdout }} - - name: Fetch log - fetch: - src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - dest: "{{ logs_path }}/" - flat: yes \ No newline at end of file + command: "bash wazuh-install.sh -wd {{ inventory_hostname }} -v" + args: + chdir: "{{ tmp_path }}" + register: dashboard diff --git a/.github/workflows/ansible-playbooks/distributed_install_indexer.yml b/.github/workflows/ansible-playbooks/distributed_install_indexer.yml index 7f41bd7..57d385e 100644 --- a/.github/workflows/ansible-playbooks/distributed_install_indexer.yml +++ b/.github/workflows/ansible-playbooks/distributed_install_indexer.yml @@ -4,41 +4,10 @@ become: true vars: - test_name: "{{ test_name }}" - test_dir: "/{{ test_name }}" - logs_path: "{{ logs_path }}" + tmp_path: "{{ tmp_path }}" tasks: - - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - state: touch - - name: Install Wazuh indexer - block: - - name: Install Wazuh indexer - command: "bash {{ tmp_path }}/wazuh-install.sh -wi {{ inventory_hostname }} -v" - register: indexer - - always: - - - name: Save output Wazuh indexer - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - block: | - {{ indexer.stderr }} - -------------------------------- - {{ indexer.stdout }} + command: "bash {{ tmp_path }}/wazuh-install.sh -wi {{ inventory_hostname }} -v" + register: indexer - - name: Fetch log - fetch: - src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - dest: "{{ logs_path }}/" - flat: yes diff --git a/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml index 26a5fae..a128a19 100644 --- a/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml +++ b/.github/workflows/ansible-playbooks/distributed_install_wazuh.yml @@ -4,40 +4,14 @@ become: true vars: - test_name: "{{ test_name }}" tmp_path: "{{ tmp_path }}" - test_dir: "/{{ test_name }}" - logs_path: "{{ logs_path }}" master_ip: "{{ hostvars[groups['managers'] | select('match', 'master') | first]['private_ip'] }}" check_port: 55000 - retries: 10 - delay: 30 tasks: - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - state: touch - - name: Install Wazuh server on master - block: - - name: Install Wazuh server (Master) - command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" - register: wazuh - - - name: Save Wazuh installation log (Master) - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - block: | - {{ wazuh.stderr }} - -------------------------------- - {{ wazuh.stdout }} + command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" + register: wazuh when: hostvars[inventory_hostname].manager_type == 'master' - name: Install Wazuh server on worker nodes @@ -46,7 +20,7 @@ wait_for: host: "{{ master_ip }}" port: "{{ check_port }}" - delay: "{{ delay }}" + delay: 30 timeout: 300 when: hostvars[inventory_hostname].manager_type == 'worker' async: 500 @@ -55,15 +29,6 @@ - name: Install Wazuh server (Workers) command: "bash {{ tmp_path }}/wazuh-install.sh -ws {{ inventory_hostname }} -v" register: wazuh - - - name: Save Wazuh installation log (Workers) - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - block: | - {{ wazuh.stderr }} - -------------------------------- - {{ wazuh.stdout }} when: hostvars[inventory_hostname].manager_type == 'worker' diff --git a/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml index 4c94f8a..37d8cbb 100644 --- a/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml +++ b/.github/workflows/ansible-playbooks/distributed_start_indexer_cluster.yml @@ -4,45 +4,13 @@ become: true vars: - test_name: "{{ test_name }}" - test_dir: "/{{ test_name }}" tmp_path: "{{ tmp_path }}" - logs_path: "{{ logs_path }}" tasks: - name: Start Wazuh indexer cluster in just one node - block: - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - state: touch - - - name: Start cluster - block: - - name: Start Wazuh indexer cluster - command: "bash {{ tmp_path }}/wazuh-install.sh -s -v" - register: cluster - - always: - - - name: Save output start cluster - blockinfile: - marker: "" - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - block: | - {{ cluster.stderr }} - -------------------------------- - {{ cluster.stdout }} - - - name: Fetch log - fetch: - src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - dest: "{{ logs_path }}/" - flat: yes + block: + - name: Start Wazuh indexer cluster + command: "bash {{ tmp_path }}/wazuh-install.sh -s -v" + register: cluster when: inventory_hostname == ansible_play_hosts[0] diff --git a/.github/workflows/ansible-playbooks/distributed_tests.yml b/.github/workflows/ansible-playbooks/distributed_tests.yml index b3fbf54..2d4cfbf 100644 --- a/.github/workflows/ansible-playbooks/distributed_tests.yml +++ b/.github/workflows/ansible-playbooks/distributed_tests.yml @@ -5,106 +5,37 @@ vars: script_path: "{{ tmp_path }}/tests/install" - test_name: "{{ test_name }}" script_name: "{{ test_name }}.py" - test_dir: "/{{ test_name }}" - logs_path: "{{ logs_path }}" tasks: - - - name: Create log directory - file: - path: "{{ test_dir }}" - state: directory - - - name: Create log file - file: - dest: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - state: touch - - - name: Test distributed master - block: - - - name: Execute tests Master - command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_cluster\"" - args: - chdir: "{{ script_path }}" - register: test_results_master - when: - - hostvars[inventory_hostname].manager_type == 'master' - - - name: Execute tests Workers - command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_worker\"" - args: - chdir: "{{ script_path }}" - register: test_results_worker - when: - - hostvars[inventory_hostname].manager_type == 'worker' - - - name: Execute tests indexer_manager - command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster\"" - args: - chdir: "{{ script_path }}" - register: test_results_indexer_manager - when: - - hostvars[inventory_hostname].instance_type == 'indexer_manager' - - - name: Execute tests indexer_manager_dashboard - command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster or dashboard\"" - args: - chdir: "{{ script_path }}" - register: test_results_indexer_manager_dashboard - when: - - hostvars[inventory_hostname].instance_type == 'indexer_manager_dashboard' - - always: - - - name: Save output Master - blockinfile: - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - marker: "" - block: | - {{ test_results_master.stderr }} - -------------------------------- - {{ test_results_master.stdout }} - when: - - hostvars[inventory_hostname].manager_type == 'master' - - - name: Save output Worker - blockinfile: - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - marker: "" - block: | - {{ test_results_worker.stderr }} - -------------------------------- - {{ test_results_worker.stdout }} - when: - - hostvars[inventory_hostname].manager_type == 'worker' - - - name: Save output indexer_manager - blockinfile: - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - marker: "" - block: | - {{ test_results_indexer_manager.stderr }} - -------------------------------- - {{ test_results_indexer_manager.stdout }} - when: - - hostvars[inventory_hostname].instance_type == 'indexer_manager' - - - name: Save output indexer_manager_dashboard - blockinfile: - path: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - marker: "" - block: | - {{ test_results_indexer_manager_dashboard.stderr }} - -------------------------------- - {{ test_results_indexer_manager_dashboard.stdout }} - when: - - hostvars[inventory_hostname].instance_type == 'indexer_manager_dashboard' - - - name: Fetch logs - fetch: - src: "{{ test_dir }}/{{ test_name }}_{{ inventory_hostname }}.log" - dest: "{{ logs_path }}/" - flat: yes \ No newline at end of file + - name: Execute tests Master + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_cluster\"" + args: + chdir: "{{ script_path }}" + register: test_results_master + when: + - hostvars[inventory_hostname].manager_type == 'master' + + - name: Execute tests Workers + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh_worker\"" + args: + chdir: "{{ script_path }}" + register: test_results_worker + when: + - hostvars[inventory_hostname].manager_type == 'worker' + + - name: Execute tests indexer_manager + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster\"" + args: + chdir: "{{ script_path }}" + register: test_results_indexer_manager + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager' + + - name: Execute tests indexer_manager_dashboard + command: "python3 -m pytest --tb=long {{ script_name }} -v -m \"wazuh or indexer or indexer_cluster or dashboard\"" + args: + chdir: "{{ script_path }}" + register: test_results_indexer_manager_dashboard + when: + - hostvars[inventory_hostname].instance_type == 'indexer_manager_dashboard' From 4864a73814f9f9d2c623d75c0f3ed20297419a6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 16 Sep 2024 17:12:49 +0200 Subject: [PATCH 16/19] Changed repository reference to the workflow branch --- .../workflows/Test_installation_assistant_distributed.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index 6e0c23c..c7b62b0 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -24,14 +24,10 @@ on: description: 'Branch or tag of the wazuh-automation repository' required: true default: '4.10.0' - ASSISTANT_REFERENCE: - description: 'Branch or tag of the wazuh-installation-assistant repository' - required: true - default: '4.10.0' SYSTEMS: description: 'Operating Systems (list of comma-separated quoted strings enclosed in square brackets)' required: true - default: '["CentOS_8", "AmazonLinux_2","Ubuntu_22", "RHEL8"]' + default: '["Ubuntu_22"]' type: string VERBOSITY: description: 'Verbosity level on playbooks execution' @@ -216,7 +212,7 @@ jobs: -i $ALLOCATOR_PATH/inventory \ -l indexers \ -e "repository=$REPOSITORY_URL" \ - -e "reference=${{ inputs.ASSISTANT_REFERENCE }}" \ + -e "reference=${{ github.ref_name }}" \ -e "tmp_path=$TMP_PATH" \ -e "pkg_repository=$PKG_REPOSITORY" \ -e "install_deps=$INSTALL_DEPS" \ From f21e89a57913f78f28eeafeaa23f8cf0f8c6404c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Mon, 16 Sep 2024 17:13:09 +0200 Subject: [PATCH 17/19] Added README for workflows --- .github/workflows/README.md | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..9381003 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,60 @@ +# Installation assistant workflows + +This repository includes several GitHub Actions workflows. These workflows are designed to automate the testing process for the installation of the Wazuh Installation Assistant in various environments and to build the different tools and scripts. + +## Workflows Overview + +1. `Test_installation_assistant`. +This workflow tests the installation of the Wazuh Installation Assistant in a single-node setup. It triggers on pull requests that modify specific directories or files, and can also be manually dispatched. + +2. `Test_installation_assistant_distributed`. +This workflow is an extension of the Test_installation_assistant workflow, intended for distributed environments. It provisions three instances and simulates a distributed Wazuh deployment across multiple nodes (indexers, managers, and dashboards). + +## Triggering the Workflows +### Automatic Trigger +The workflows tests are triggered automatically when a pull request (PR) is created or updated, affecting the following paths: + +- `cert_tool/` +- `common_functions/` +- `config/` +- `install_functions/` +- `passwords_tool/` +- `tests/` + +### Manual Trigger +The test workflows can be triggered manually via the GitHub interface under the "Actions" tab, using the workflow_dispatch event. When triggered manually, several input parameters are required: + +- **REPOSITORY**: Defines the repository environment (e.g., staging, pre-release). +- **AUTOMATION_REFERENCE**: The branch or tag of the `wazuh-automation` repository, used to clone the Allocation module. +- **SYSTEMS**: A comma-separated list of operating systems to be tested, enclosed in square brackets (e.g., `["CentOS_8", "AmazonLinux_2", "Ubuntu_22", "RHEL8"]`). The available options are: `CentOS_7`, `CentOS_8`, `AmazonLinux_2`, `Ubuntu_16`, `Ubuntu_18`, `Ubuntu_20`, `Ubuntu_22`, `RHEL7`, `RHEL8`. +- **VERBOSITY**: The verbosity level for Ansible playbook execution, with options `-v`, `-vv`, `-vvv`, and `-vvvv`. +- **DESTROY**: Boolean value (true or false) indicating whether to destroy the instances after testing. + +## Workflow Structure +### Jobs + +The tests workflows follow a similar structure with the following key jobs: + +1. **Checkout Code**: The workflow fetches the latest code from the wazuh-automation and wazuh-installation-assistant repositories. + +2. **Set Up Environment**: The operating system is configured based on the selected OS in the SYSTEMS input. The corresponding OS name is stored in the environment variable COMPOSITE_NAME. + +3. **Install Ansible**: Ansible is installed for managing the provisioning of instances and running the necessary playbooks. + +4. **Provisioning Instances**: The distributed workflow allocates AWS instances using the wazuh-automation repository’s allocation module. It provisions indexers, managers, and dashboards across the instances. The instance inventory is dynamically created and used for later playbook executions. + +5. **Ansible Playbooks Execution**: Provision playbooks are executed to prepare the environments for Wazuh components. + +6. **Test Execution**: A Python-based testing framework is executed to verify the successful installation and functionality of the Wazuh components on the allocated instances. + +7. **Destroy Instances (Optional)**: If the `DESTROY` parameter is set to true, the allocated AWS instances are terminated after the tests. If set to false, the instances and their details are saved as artifacts for later analysis. + +### Artifacts +If instances are not destroyed, the workflow compresses the allocated instances' directory and uploads it as an artifact. Also, the artifacts are compressed with a password. Ask @devel-devops teams for this password. An artifact is uploaded per OS selected. +## Notes +- Instance allocation: The `Test_installation_assistant_distributed` workflow provisions three instances by default. The roles are distributed as follows: + - `indexer1`, `indexer2`, `indexer3`: Indexers in the Wazuh cluster. + - `master`, `worker1`, `worker2`: Wazuh managers, where `master` is the main manager, and `worker1` and `worker2` are worker nodes. + - `dashboard`: Wazuh dashboard. + +- Customization: These workflows allow for customization through the various input parameters, making it easy to test different operating systems, verbosity levels, or different versions of the repositories. \ No newline at end of file From cccd73503b7fb42f1a66a8d771f27a2b2544923e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 17 Sep 2024 10:02:15 +0200 Subject: [PATCH 18/19] Updated CHANGELOG for #60 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85e2091..7063012 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Changed - Added post-install validations for the Wazuh manager and Filebeat. ([#3059](https://github.com/wazuh/wazuh-packages/pull/3059)) +- Installation assistant distributed test rework and migration. ([#60](https://github.com/wazuh/wazuh-installation-assistant/pull/60)) ### Fixed From 2b0dacd079644f8c1cb4e3c8ee42c96fb95210d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Correa=20Rodr=C3=ADguez?= Date: Tue, 17 Sep 2024 14:19:11 +0200 Subject: [PATCH 19/19] Changing systems when PR is created --- .github/workflows/Test_installation_assistant_distributed.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/Test_installation_assistant_distributed.yml b/.github/workflows/Test_installation_assistant_distributed.yml index c7b62b0..9170169 100644 --- a/.github/workflows/Test_installation_assistant_distributed.yml +++ b/.github/workflows/Test_installation_assistant_distributed.yml @@ -27,7 +27,7 @@ on: SYSTEMS: description: 'Operating Systems (list of comma-separated quoted strings enclosed in square brackets)' required: true - default: '["Ubuntu_22"]' + default: '["CentOS_8", "AmazonLinux_2","Ubuntu_22", "RHEL8"]' type: string VERBOSITY: description: 'Verbosity level on playbooks execution' @@ -68,7 +68,7 @@ jobs: strategy: fail-fast: false # If a job fails, the rest of jobs will not be canceled matrix: - system: ${{ fromJson(inputs.SYSTEMS) }} + system: ${{ github.event_name == 'pull_request' && '["Ubuntu_22"]' || fromJson(inputs.SYSTEMS) }} # If the worklflow is executed by a PR, set the OSs steps: - name: Checkout code