From 358c212b0de1fe16e1c0155a0f38ddbb95b617af Mon Sep 17 00:00:00 2001 From: Tomas Jelinek Date: Wed, 20 Mar 2024 12:33:10 +0100 Subject: [PATCH] move inventory variables to playbook --- .gitlab-ci.yml | 8 +- README.md | 175 ++++++++++++++---- examples/sbd.yml | 22 ++- .../check-and-prepare-role-variables.yml | 56 +++--- tasks/shell_pcs/cluster-auth-pcs-0.10.yml | 4 +- .../shell_pcs/pcs-cluster-setup-pcs-0.10.yml | 3 +- tasks/shell_pcs/sbd.yml | 19 +- tests/template_sbd_all_options.yml | 133 +++++++++++++ tests/tests_sbd_all_options.yml | 164 ---------------- tests/tests_sbd_all_options_combined.yml | 77 ++++++++ tests/tests_sbd_all_options_inventory.yml | 46 +++++ tests/tests_sbd_all_options_play.yml | 52 ++++++ ...tests_sbd_check_devices_count_combined.yml | 82 ++++++++ ...sts_sbd_check_devices_count_inventory.yml} | 7 +- tests/tests_sbd_check_devices_count_play.yml | 68 +++++++ 15 files changed, 682 insertions(+), 234 deletions(-) create mode 100644 tests/template_sbd_all_options.yml delete mode 100644 tests/tests_sbd_all_options.yml create mode 100644 tests/tests_sbd_all_options_combined.yml create mode 100644 tests/tests_sbd_all_options_inventory.yml create mode 100644 tests/tests_sbd_all_options_play.yml create mode 100644 tests/tests_sbd_check_devices_count_combined.yml rename tests/{tests_sbd_check_devices_count.yml => tests_sbd_check_devices_count_inventory.yml} (80%) create mode 100644 tests/tests_sbd_check_devices_count_play.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79fd027f..c89f2fbf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -59,8 +59,12 @@ variables: - qnetd_and_cluster - qnetd_disabled - qnetd - - sbd_all_options - - sbd_check_devices_count + - sbd_all_options_combined + - sbd_all_options_inventory + - sbd_all_options_play + - sbd_check_devices_count_combined + - sbd_check_devices_count_inventory + - sbd_check_devices_count_play - sbd_defaults_disabled - sbd_defaults - sbd_delay_start diff --git a/README.md b/README.md index 75c1fccc..2ba07aa6 100644 --- a/README.md +++ b/README.md @@ -485,26 +485,9 @@ options are: You may take a look at [an example](#configuring-cluster-to-use-sbd). Watchdog and SBD devices are configured on a node to node basis in +[`ha_cluster_node_options`](#ha_cluster_node_options) (preferred) or [inventory](#sbd-watchdog-and-devices). -#### `ha_cluster_cluster_properties` - -structure, default: no properties - -```yaml -ha_cluster_cluster_properties: - - attrs: - - name: property1_name - value: property1_value - - name: property2_name - value: property2_value -``` - -List of sets of cluster properties - Pacemaker cluster-wide configuration. -Currently, only one set is supported. - -You may take a look at [an example](#configuring-cluster-properties). - #### `ha_cluster_node_options` structure, default: no node options @@ -512,6 +495,19 @@ structure, default: no node options ```yaml ha_cluster_node_options: - node_name: node1 + pcs_address: node1-address + corosync_addresses: + - 192.168.1.11 + - 192.168.2.11 + sbd_watchdog_modules: + - module1 + - module2 + sbd_watchdog_modules_blocklist: + - module3 + sbd_watchdog: /dev/watchdog2 + sbd_devices: + - /dev/vdx + - /dev/vdy attributes: - attrs: - name: attribute1 @@ -519,6 +515,18 @@ ha_cluster_node_options: - name: attribute2 value: value2_node1 - node_name: node2 + pcs_address: node2-address:2224 + corosync_addresses: + - 192.168.1.12 + - 192.168.2.12 + sbd_watchdog_modules: + - module1 + sbd_watchdog_modules_blocklist: + - module3 + sbd_watchdog: /dev/watchdog1 + sbd_devices: + - /dev/vdw + - /dev/vdz attributes: - attrs: - name: attribute1 @@ -528,18 +536,54 @@ ha_cluster_node_options: ``` This variable defines various settings which vary from cluster node to cluster -node. **Note:** Use an inventory or playbook hosts to specify which nodes form -the cluster. This variable merely sets options for the specified nodes. The -items are as follows: +node. + +**Note:** Use an inventory or playbook hosts to specify which nodes form +the cluster. This variable merely sets options for the specified nodes. + +The items are as follows: -* `node_name` (mandatory) - Node name. +* `node_name` (mandatory) - Node name. It must match a name defined for a node + in [inventory variable `ha_cluster.node_name`](#nodes-names-and-addresses). +* `pcs_address` (optional) - Address used by pcs to communicate with the node, + it can be a name, a FQDN or an IP address. Port can be specified as well. +* `corosync_addresses` (optional) - List of addresses used by Corosync, all + nodes must have the same number of addresses and the order of the addresses + matters. +* `sbd_watchdog_modules` (optional) - Watchdog kernel modules to be loaded + (creates `/dev/watchdog*` devices). Defaults to empty list if not set. +* `sbd_watchdog_modules_blocklist` (optional) - Watchdog kernel modules to be + unloaded and blocked. Defaults to empty list if not set. +* `sbd_watchdog` (optional) - Watchdog device to be used by SBD. Defaults to + `/dev/watchdog` if not set. +* `sbd_devices` (optional) - Devices to use for exchanging SBD messages and for + monitoring. Defaults to empty list if not set. * `attributes` (optional) - List of sets of Pacemaker node attributes for the node. Currently, no more than one set for each node is supported. You may take a look at examples: +* [configuring cluster to use SBD](#configuring-cluster-to-use-sbd) * [configuring node attributes](#configuring-node-attributes) +#### `ha_cluster_cluster_properties` + +structure, default: no properties + +```yaml +ha_cluster_cluster_properties: + - attrs: + - name: property1_name + value: property1_value + - name: property2_name + value: property2_value +``` + +List of sets of cluster properties - Pacemaker cluster-wide configuration. +Currently, only one set is supported. + +You may take a look at [an example](#configuring-cluster-properties). + #### `ha_cluster_resource_primitives` structure, default: no resources @@ -1303,8 +1347,10 @@ example](#configuring-a-cluster-using-a-quorum-device). #### Nodes' names and addresses -Nodes' names and addresses can be configured in inventory. This is optional. If -no names or addresses are configured, play's targets will be used. +Nodes' names and addresses can be configured in inventory. This is optional. +Addresses configured in [`ha_cluster_node_options`](#ha_cluster_node_options) +override those configured in inventory. +If no names or addresses are configured, play's targets will be used. Example inventory with targets `node1` and `node2`: @@ -1338,8 +1384,10 @@ all: When using SBD, you may optionally configure watchdog and SBD devices for each node in inventory. Even though all SBD devices must be shared to and accessible from all nodes, each node may use different names for the devices. The loaded -watchdog modules and used devices may also be different for each node. See also -[SBD variables](#ha_cluster_sbd_enabled). +watchdog modules and used devices may also be different for each node. +SBD settings defined in [`ha_cluster_node_options`](#ha_cluster_node_options) +override those defined in inventory. +See also [SBD variables](#ha_cluster_sbd_enabled). Example inventory with targets `node1` and `node2`: @@ -1484,10 +1532,70 @@ in /var/lib/pcsd with the file name FILENAME.crt and FILENAME.key, respectively. ### Configuring cluster to use SBD -#### inventory +#### Using playbook variables + +```yaml +- hosts: node1 node2 + vars: + ha_cluster_cluster_name: my-new-cluster + ha_cluster_hacluster_password: password + ha_cluster_sbd_enabled: true + ha_cluster_sbd_options: + - name: delay-start + value: 'no' + - name: startmode + value: always + - name: timeout-action + value: 'flush,reboot' + - name: watchdog-timeout + value: 30 + ha_cluster_node_options: + - node_name: node1 + sbd_watchdog_modules: + - iTCO_wdt + sbd_watchdog_modules_blocklist: + - ipmi_watchdog + sbd_watchdog: /dev/watchdog1 + sbd_devices: + - /dev/vdx + - /dev/vdy + - /dev/vdz + - node_name: node2 + sbd_watchdog_modules: + - iTCO_wdt + sbd_watchdog_modules_blocklist: + - ipmi_watchdog + sbd_watchdog: /dev/watchdog1 + sbd_devices: + - /dev/vdx + - /dev/vdy + - /dev/vdz + # Best practice for setting SBD timeouts: + # watchdog-timeout * 2 = msgwait-timeout (set automatically) + # msgwait-timeout * 1.2 = stonith-timeout + ha_cluster_cluster_properties: + - attrs: + - name: stonith-timeout + value: 72 + ha_cluster_resource_primitives: + - id: fence_sbd + agent: 'stonith:fence_sbd' + instance_attrs: + - attrs: + # taken from host_vars + - name: devices + value: "{{ ha_cluster.sbd_devices | join(',') }}" + - name: pcmk_delay_base + value: 30 + + roles: + - linux-system-roles.ha_cluster +``` -These variables need to be set in inventory or via `host_vars`. Of course -the SBD kernel modules and device path might differ depending on your setup. +#### Using inventory + +The same result can be achieved by specifying node-specific options in inventory +like this: ```yaml all: @@ -1516,11 +1624,7 @@ all: - /dev/vdz ``` -#### playbook - -After setting the inventory correctly, use this playbook to configure a -complete SBD setup including loading watchdog modules and creating the -SBD stonith resource. +Variables specified in inventory can be omitted when writing the playbook: ```yaml - hosts: node1 node2 @@ -1559,6 +1663,9 @@ SBD stonith resource. - linux-system-roles.ha_cluster ``` +If both the inventory and the playbook contain SBD options, those in the +playbook have precedence. + ### Configuring cluster properties ```yaml diff --git a/examples/sbd.yml b/examples/sbd.yml index 8ef33a53..31b9d476 100644 --- a/examples/sbd.yml +++ b/examples/sbd.yml @@ -2,7 +2,6 @@ --- - name: Example ha_cluster role invocation - cluster with SBD hosts: node1 node2 - # do not forget to also set variables via inventory or host_vars. vars: ha_cluster_manage_firewall: true ha_cluster_manage_selinux: true @@ -18,6 +17,27 @@ value: 'flush,reboot' - name: watchdog-timeout value: 30 + ha_cluster_node_options: + - node_name: node1 + sbd_watchdog_modules: + - iTCO_wdt + sbd_watchdog_modules_blocklist: + - ipmi_watchdog + sbd_watchdog: /dev/watchdog1 + sbd_devices: + - /dev/vdx + - /dev/vdy + - /dev/vdz + - node_name: node2 + sbd_watchdog_modules: + - iTCO_wdt + sbd_watchdog_modules_blocklist: + - ipmi_watchdog + sbd_watchdog: /dev/watchdog1 + sbd_devices: + - /dev/vdx + - /dev/vdy + - /dev/vdz # Best practice for setting SBD timeouts: # watchdog-timeout * 2 = msgwait-timeout (set automatically) # msgwait-timeout * 1.2 = stonith-timeout diff --git a/tasks/shell_pcs/check-and-prepare-role-variables.yml b/tasks/shell_pcs/check-and-prepare-role-variables.yml index e7185fa1..ae7d7d69 100644 --- a/tasks/shell_pcs/check-and-prepare-role-variables.yml +++ b/tasks/shell_pcs/check-and-prepare-role-variables.yml @@ -12,6 +12,37 @@ | list }}" +- name: Check ha_cluster_node_options + run_once: true + vars: + __nodes_from_options: "{{ + ha_cluster_node_options | map(attribute='node_name') | list }}" + block: + - name: > + Fail if ha_cluster_node_options contains unknown or duplicate nodes + fail: + msg: > + node_name fields in ha_cluster_node_options must be unique + and they must match cluster nodes + when: + - > + ( + __nodes_from_options != (__nodes_from_options | unique) + ) or ( + __nodes_from_options | difference(__ha_cluster_all_node_names) + ) + +- name: Extract node options + set_fact: + __ha_cluster_local_node: >- + {{ + (ha_cluster | d({})) | combine( + (ha_cluster_node_options | d([])) + | selectattr('node_name', 'match', '^' ~ __ha_cluster_node_name ~ '$') + | list | last | d({}) + ) + }} + - name: Check cluster configuration variables block: - name: Fail if passwords are not specified @@ -32,7 +63,8 @@ - ha_cluster_sbd_enabled - > ansible_play_hosts - | map('extract', hostvars, ['ha_cluster', 'sbd_devices']) + | map('extract', hostvars, + ['__ha_cluster_local_node', 'sbd_devices']) | map('default', [], true) | map('length') | unique | length > 1 run_once: true @@ -71,26 +103,6 @@ loop: "{{ ha_cluster_stonith_levels }}" run_once: true - - name: Check ha_cluster_node_options - run_once: true - vars: - __nodes_from_options: "{{ - ha_cluster_node_options | map(attribute='node_name') | list }}" - block: - - name: > - Fail if ha_cluster_node_options contains unknown or duplicate nodes - fail: - msg: > - node_name fields in ha_cluster_node_options must be unique - and they must match cluster nodes - when: - - > - ( - __nodes_from_options != (__nodes_from_options | unique) - ) or ( - __nodes_from_options | difference(__ha_cluster_all_node_names) - ) - - name: Extract qdevice settings set_fact: __ha_cluster_qdevice_in_use: "{{ 'device' in ha_cluster_quorum }}" @@ -118,7 +130,7 @@ # - qdevice is not used __ha_cluster_sbd_needs_atb: "{{ ha_cluster_sbd_enabled - and not ha_cluster.sbd_devices | d([]) + and not __ha_cluster_local_node.sbd_devices | d([]) and __ha_cluster_all_node_names | length is even and not __ha_cluster_qdevice_in_use }}" diff --git a/tasks/shell_pcs/cluster-auth-pcs-0.10.yml b/tasks/shell_pcs/cluster-auth-pcs-0.10.yml index c3a59d27..ee3937f7 100644 --- a/tasks/shell_pcs/cluster-auth-pcs-0.10.yml +++ b/tasks/shell_pcs/cluster-auth-pcs-0.10.yml @@ -8,8 +8,8 @@ pcs host auth -u hacluster -- {% for node in ansible_play_hosts %} {{ hostvars[node].__ha_cluster_node_name | quote }} - {% if hostvars[node].ha_cluster.pcs_address | d("") %} - addr={{ hostvars[node].ha_cluster.pcs_address | quote }} + {% if hostvars[node].__ha_cluster_local_node.pcs_address | d("") %} + addr={{ hostvars[node].__ha_cluster_local_node.pcs_address | quote }} {% endif %} {% endfor %} stdin: "{{ ha_cluster_hacluster_password }}" diff --git a/tasks/shell_pcs/pcs-cluster-setup-pcs-0.10.yml b/tasks/shell_pcs/pcs-cluster-setup-pcs-0.10.yml index e10f9164..a97fb1ff 100644 --- a/tasks/shell_pcs/pcs-cluster-setup-pcs-0.10.yml +++ b/tasks/shell_pcs/pcs-cluster-setup-pcs-0.10.yml @@ -14,7 +14,8 @@ {% for node in ansible_play_hosts %} {{ hostvars[node].__ha_cluster_node_name | quote }} {% - for addr in hostvars[node].ha_cluster.corosync_addresses | d([]) + for addr in + hostvars[node].__ha_cluster_local_node.corosync_addresses | d([]) %} addr={{ addr | quote }} {% endfor %} diff --git a/tasks/shell_pcs/sbd.yml b/tasks/shell_pcs/sbd.yml index 89b0eba7..41bb95c9 100644 --- a/tasks/shell_pcs/sbd.yml +++ b/tasks/shell_pcs/sbd.yml @@ -17,13 +17,17 @@ # wokeignore:rule=blacklist line: "blacklist {{ item }}" state: present - loop: "{{ ha_cluster.sbd_watchdog_modules_blocklist | d([]) }}" + loop: "{{ + __ha_cluster_local_node.sbd_watchdog_modules_blocklist | d([]) + }}" - name: Unload watchdog kernel modules from blocklist community.general.modprobe: name: "{{ item }}" state: absent - loop: "{{ ha_cluster.sbd_watchdog_modules_blocklist | d([]) }}" + loop: "{{ + __ha_cluster_local_node.sbd_watchdog_modules_blocklist | d([]) + }}" - name: Configure and load watchdog kernel module block: @@ -35,13 +39,13 @@ regexp: "^{{ item }}" line: "{{ item }}" state: present - loop: "{{ ha_cluster.sbd_watchdog_modules | d([]) }}" + loop: "{{ __ha_cluster_local_node.sbd_watchdog_modules | d([]) }}" - name: Load watchdog kernel modules community.general.modprobe: name: "{{ item }}" state: present - loop: "{{ ha_cluster.sbd_watchdog_modules | d([]) }}" + loop: "{{ __ha_cluster_local_node.sbd_watchdog_modules | d([]) }}" - name: Manage SBD devices # Ideally, the block as a whole should run one node at a time. This does @@ -57,7 +61,7 @@ - name: Probe SBD devices command: cmd: sbd -d {{ item | quote }} dump - loop: "{{ ha_cluster.sbd_devices | d([]) }}" + loop: "{{ __ha_cluster_local_node.sbd_devices | d([]) }}" register: __ha_cluster_check_sbd_devices_result changed_when: false # return_code == 0 means the disk is initialized already @@ -95,8 +99,9 @@ vars: options: "{{ ha_cluster_sbd_options | d([]) }}" node_name: "{{ __ha_cluster_node_name }}" - node_watchdog: "{{ ha_cluster.sbd_watchdog | d('/dev/watchdog') }}" - node_devices: "{{ ha_cluster.sbd_devices | d([]) }}" + node_watchdog: "{{ __ha_cluster_local_node.sbd_watchdog + | d('/dev/watchdog') }}" + node_devices: "{{ __ha_cluster_local_node.sbd_devices | d([]) }}" register: __ha_cluster_distribute_sbd_config - name: Configure systemd timeout for SBD diff --git a/tests/template_sbd_all_options.yml b/tests/template_sbd_all_options.yml new file mode 100644 index 00000000..d2c005bb --- /dev/null +++ b/tests/template_sbd_all_options.yml @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: MIT +--- +- name: Run test + vars: + ha_cluster_cluster_name: test-cluster + ha_cluster_sbd_enabled: true + ha_cluster_sbd_options: + - name: delay-start + value: 2 + - name: startmode + value: clean + - name: timeout-action + value: reboot,flush + - name: watchdog-timeout + value: 10 + block: + - name: Ensure modprobe files are not present + file: + state: absent + path: "{{ item }}" + loop: + - /etc/modprobe.d/iTCO_wdt.conf + - /etc/modules-load.d/softdog.conf + + - name: Run HA Cluster role + include_role: + name: linux-system-roles.ha_cluster + public: true + + - name: Slurp generated SBD watchdog blocklist file + slurp: + src: /etc/modprobe.d/iTCO_wdt.conf + register: __test_sbd_watchdog_blocklist_file + + - name: Decode SBD watchdog blocklist file + set_fact: + __test_sbd_watchdog_blocklist_file_lines: "{{ + (__test_sbd_watchdog_blocklist_file.content | + b64decode).splitlines() + }}" + + - name: Print SBD watchdog blocklist file lines + debug: + var: __test_sbd_watchdog_blocklist_file_lines + + - name: Check SBD watchdog blocklist file + assert: + that: + - __blockstr in __test_sbd_watchdog_blocklist_file_lines + vars: + # wokeignore:rule=blacklist + __blockstr: blacklist iTCO_wdt + + - name: Slurp generated SBD watchdog modprobe file + slurp: + src: /etc/modules-load.d/softdog.conf + register: __test_sbd_watchdog_modprobe_file + + - name: Decode SBD watchdog modprobe file + set_fact: + __test_sbd_watchdog_modprobe_file_lines: "{{ + (__test_sbd_watchdog_modprobe_file.content | + b64decode).splitlines() + }}" + + - name: Print SBD watchdog modprobe file lines + debug: + var: __test_sbd_watchdog_modprobe_file_lines + + - name: Check SBD watchdog modprobe file + assert: + that: + - __modulestr in __test_sbd_watchdog_modprobe_file_lines + vars: + __modulestr: softdog + + - name: Run lsmod for SBD watchdog module + command: lsmod + changed_when: false + register: __test_sbd_watchdog_sbd_lsmod + + - name: Print lsmod + debug: + var: __test_sbd_watchdog_sbd_lsmod + + - name: Check lsmod output for absence of SBD watchdog module blocklist + assert: + that: + - "'iTCO_wdt' not in __test_sbd_watchdog_sbd_lsmod.stdout" + + - name: Check lsmod output for SBD watchdog module + assert: + that: + - "'softdog' in __test_sbd_watchdog_sbd_lsmod.stdout" + + - name: Slurp SBD config file + slurp: + src: /etc/sysconfig/sbd + register: __test_sbd_config + + - name: Decode SBD config + set_fact: + __test_sbd_config_lines: "{{ + (__test_sbd_config.content | b64decode).splitlines() + }}" + + - name: Print SBD config lines + debug: + var: __test_sbd_config_lines + + - name: Check SBD config + assert: + that: + - "'SBD_DELAY_START=\"2\"' in __test_sbd_config_lines" + - "'SBD_DEVICE=\"' ~ __test_sbd_mount.stdout ~ '\"' + in __test_sbd_config_lines" + - "'SBD_STARTMODE=\"clean\"' in __test_sbd_config_lines" + - "'SBD_TIMEOUT_ACTION=\"reboot,flush\"' + in __test_sbd_config_lines" + - "'SBD_WATCHDOG_DEV=\"/dev/null\"' in __test_sbd_config_lines" + - "'SBD_WATCHDOG_TIMEOUT=\"10\"' in __test_sbd_config_lines" + - > + __test_sbd_config_lines[-1] + == 'SBD_OPTS="-n ' ~ __ha_cluster_node_name ~ '"' + + - name: Check firewall and selinux state + include_tasks: tasks/check_firewall_selinux.yml + + - name: Check header for ansible_managed, fingerprint + include_tasks: tasks/check_header.yml + vars: + __file_content: "{{ __test_sbd_config }}" + __fingerprint: "system_role:ha_cluster" diff --git a/tests/tests_sbd_all_options.yml b/tests/tests_sbd_all_options.yml deleted file mode 100644 index 8484c0e4..00000000 --- a/tests/tests_sbd_all_options.yml +++ /dev/null @@ -1,164 +0,0 @@ -# SPDX-License-Identifier: MIT ---- -- name: Maximal SBD configuration - hosts: all - vars_files: vars/main.yml - vars: - ha_cluster_cluster_name: test-cluster - ha_cluster_sbd_enabled: true - ha_cluster_sbd_options: - - name: delay-start - value: 2 - - name: startmode - value: clean - - name: timeout-action - value: reboot,flush - - name: watchdog-timeout - value: 10 - - tasks: - - name: Run test - tags: tests::verify - block: - - name: Set up test environment - include_role: - name: linux-system-roles.ha_cluster - tasks_from: test_setup.yml - - - name: Set up test environment for SBD - include_role: - name: linux-system-roles.ha_cluster - tasks_from: test_setup_sbd.yml - vars: - __test_disable_modprobe: true # noqa: var-naming[no-role-prefix] - - - name: Set SBD devices and watchdogs - set_fact: - ha_cluster: - sbd_watchdog_modules: - - softdog - sbd_watchdog_modules_blocklist: - - iTCO_wdt - sbd_watchdog: /dev/null - sbd_devices: - - "{{ __test_sbd_mount.stdout }}" - - - name: Run HA Cluster role - include_role: - name: linux-system-roles.ha_cluster - public: true - - - name: Slurp generated SBD watchdog blocklist file - slurp: - src: /etc/modprobe.d/iTCO_wdt.conf - register: __test_sbd_watchdog_blocklist_file - - - name: Decode SBD watchdog blocklist file - set_fact: - __test_sbd_watchdog_blocklist_file_lines: "{{ - (__test_sbd_watchdog_blocklist_file.content | - b64decode).splitlines() - }}" - - - name: Print SBD watchdog blocklist file lines - debug: - var: __test_sbd_watchdog_blocklist_file_lines - - - name: Check SBD watchdog blocklist file - assert: - that: - - __blockstr in __test_sbd_watchdog_blocklist_file_lines - vars: - # wokeignore:rule=blacklist - __blockstr: blacklist iTCO_wdt - - - name: Slurp generated SBD watchdog modprobe file - slurp: - src: /etc/modules-load.d/softdog.conf - register: __test_sbd_watchdog_modprobe_file - - - name: Decode SBD watchdog modprobe file - set_fact: - __test_sbd_watchdog_modprobe_file_lines: "{{ - (__test_sbd_watchdog_modprobe_file.content | - b64decode).splitlines() - }}" - - - name: Print SBD watchdog modprobe file lines - debug: - var: __test_sbd_watchdog_modprobe_file_lines - - - name: Check SBD watchdog modprobe file - assert: - that: - - __modulestr in __test_sbd_watchdog_modprobe_file_lines - vars: - __modulestr: softdog - - - name: Run lsmod for SBD watchdog module - command: lsmod - changed_when: false - register: __test_sbd_watchdog_sbd_lsmod - - - name: Print lsmod - debug: - var: __test_sbd_watchdog_sbd_lsmod - - - name: Check lsmod output for absence of SBD watchdog module blocklist - assert: - that: - - "'iTCO_wdt' not in __test_sbd_watchdog_sbd_lsmod.stdout" - - - name: Check lsmod output for SBD watchdog module - assert: - that: - - "'softdog' in __test_sbd_watchdog_sbd_lsmod.stdout" - - - name: Slurp SBD config file - slurp: - src: /etc/sysconfig/sbd - register: __test_sbd_config - - - name: Decode SBD config - set_fact: - __test_sbd_config_lines: "{{ - (__test_sbd_config.content | b64decode).splitlines() - }}" - - - name: Print SBD config lines - debug: - var: __test_sbd_config_lines - - - name: Check SBD config - assert: - that: - - "'SBD_DELAY_START=\"2\"' in __test_sbd_config_lines" - - "'SBD_DEVICE=\"' ~ __test_sbd_mount.stdout ~ '\"' - in __test_sbd_config_lines" - - "'SBD_STARTMODE=\"clean\"' in __test_sbd_config_lines" - - "'SBD_TIMEOUT_ACTION=\"reboot,flush\"' - in __test_sbd_config_lines" - - "'SBD_WATCHDOG_DEV=\"/dev/null\"' in __test_sbd_config_lines" - - "'SBD_WATCHDOG_TIMEOUT=\"10\"' in __test_sbd_config_lines" - - > - __test_sbd_config_lines[-1] - == 'SBD_OPTS="-n ' ~ __ha_cluster_node_name ~ '"' - - - name: Check firewall and selinux state - include_tasks: tasks/check_firewall_selinux.yml - - - name: Check header for ansible_managed, fingerprint - include_tasks: tasks/check_header.yml - vars: - __file_content: "{{ __test_sbd_config }}" - __fingerprint: "system_role:ha_cluster" - - always: - - name: Unset SBD devices and watchdogs - set_fact: - ha_cluster: - - - name: Clean up test environment for SBD - include_role: - name: linux-system-roles.ha_cluster - tasks_from: test_cleanup_sbd.yml diff --git a/tests/tests_sbd_all_options_combined.yml b/tests/tests_sbd_all_options_combined.yml new file mode 100644 index 00000000..a09cb7cb --- /dev/null +++ b/tests/tests_sbd_all_options_combined.yml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: MIT +--- +# Test SBD configuration defined both in a play and in an inventroy. Those +# variables are merged, with variables defined in a play override variables +# from an inventory. +- name: Maximal SBD configuration (inventory + play) + hosts: all + vars_files: vars/main.yml + + tasks: + - name: Run test + tags: tests::verify + block: + - name: Set up test environment + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup.yml + + - name: Set up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup_sbd.yml + vars: + __test_disable_modprobe: true # noqa: var-naming[no-role-prefix] + + # module is defined in an inventory, module blocklist in a play, + # watchdog and devices in both + + # Generate an "inventory" variable for each node. + - name: Set SBD devices and watchdogs + set_fact: + ha_cluster: + sbd_watchdog_modules: + - softdog + sbd_watchdog: /dev/nullX + sbd_devices: + - "{{ __test_sbd_mount.stdout }}" + + # Generate a play variable. Play variables are equal for all nodes. + # This variable is expected to hold data for all nodes. + - name: Generate SBD devices and watchdogs variables + set_fact: + __test_node_options: >- + {{ + (__test_node_options | d([])) + [{ + 'node_name': item, + 'sbd_watchdog_modules_blocklist': ['iTCO_wdt'], + 'sbd_watchdog': '/dev/null', + 'sbd_devices': [__test_sbd_mount.stdout], + }] + }} + loop: "{{ ansible_play_hosts_all }}" + + # Expected values are: + # sbd_watchdog_modules: + # - softdog # defined in inventory only + # sbd_watchdog_modules_blocklist: + # - iTCO_wdt # defined in play only + # sbd_watchdog: /dev/null # play overrides inventory + # sbd_devices: + # - "{{ __test_sbd_mount.stdout }}" # defined in both + + - name: Run the role and assert results + include_tasks: template_sbd_all_options.yml + vars: + ha_cluster_node_options: "{{ __test_node_options }}" + + always: + - name: Unset SBD devices and watchdog variables + set_fact: + ha_cluster: + __test_node_options: + + - name: Clean up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_cleanup_sbd.yml diff --git a/tests/tests_sbd_all_options_inventory.yml b/tests/tests_sbd_all_options_inventory.yml new file mode 100644 index 00000000..4cf9a150 --- /dev/null +++ b/tests/tests_sbd_all_options_inventory.yml @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: MIT +--- +- name: Maximal SBD configuration (inventory) + hosts: all + vars_files: vars/main.yml + + tasks: + - name: Run test + tags: tests::verify + block: + - name: Set up test environment + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup.yml + + - name: Set up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup_sbd.yml + vars: + __test_disable_modprobe: true # noqa: var-naming[no-role-prefix] + + # Generate an "inventory" variable for each node. + - name: Set SBD devices and watchdogs + set_fact: + ha_cluster: + sbd_watchdog_modules: + - softdog + sbd_watchdog_modules_blocklist: + - iTCO_wdt + sbd_watchdog: /dev/null + sbd_devices: + - "{{ __test_sbd_mount.stdout }}" + + - name: Run the role and assert results + include_tasks: template_sbd_all_options.yml + + always: + - name: Unset SBD devices and watchdogs + set_fact: + ha_cluster: + + - name: Clean up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_cleanup_sbd.yml diff --git a/tests/tests_sbd_all_options_play.yml b/tests/tests_sbd_all_options_play.yml new file mode 100644 index 00000000..e147f0ba --- /dev/null +++ b/tests/tests_sbd_all_options_play.yml @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: MIT +--- +- name: Maximal SBD configuration (play) + hosts: all + vars_files: vars/main.yml + + tasks: + - name: Run test + tags: tests::verify + block: + - name: Set up test environment + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup.yml + + - name: Set up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup_sbd.yml + vars: + __test_disable_modprobe: true # noqa: var-naming[no-role-prefix] + + # Generate a play variable. Play variables are equal for all nodes. + # This variable is expected to hold data for all nodes. + - name: Generate SBD devices and watchdogs variables + set_fact: + __test_node_options: >- + {{ + (__test_node_options | d([])) + [{ + 'node_name': item, + 'sbd_watchdog_modules': ['softdog'], + 'sbd_watchdog_modules_blocklist': ['iTCO_wdt'], + 'sbd_watchdog': '/dev/null', + 'sbd_devices': [__test_sbd_mount.stdout], + }] + }} + loop: "{{ ansible_play_hosts_all }}" + + - name: Run the role and assert results + include_tasks: template_sbd_all_options.yml + vars: + ha_cluster_node_options: "{{ __test_node_options }}" + + always: + - name: Unset SBD devices and watchdog variables + set_fact: + __test_node_options: + + - name: Clean up test environment for SBD + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_cleanup_sbd.yml diff --git a/tests/tests_sbd_check_devices_count_combined.yml b/tests/tests_sbd_check_devices_count_combined.yml new file mode 100644 index 00000000..6c007920 --- /dev/null +++ b/tests/tests_sbd_check_devices_count_combined.yml @@ -0,0 +1,82 @@ +# SPDX-License-Identifier: MIT +--- +# All nodes must have the same number of SBD devices. The role is supposed to +# exit with an error if that's not the case. This test verifies, that the check +# works, when SBD devices are defined in both 'ha_cluster_node_options' variable +# (playbook style) and 'ha_cluster' variable (inventory style). +- name: Test a check SBD devices count matches for all nodes (inventory + play) + hosts: all + vars_files: vars/main.yml + vars: + ha_cluster_sbd_enabled: true + + tasks: + # Generate a play variable. Play variables are equal for all nodes. + # This variable is expected to hold data for all nodes. + # Number of SBD devices is different for each node. + - name: Generate SBD devices variable + set_fact: + __test_node_options: >- + {{ + (__test_node_options | d([])) + [{ + 'node_name': item, + 'sbd_devices': ( + range(1, ansible_play_hosts_all.index(item) + 2) + | product(['/tmp/dev/sdx']) | map('reverse') | map('join') + | list) + }] + }} + loop: "{{ ansible_play_hosts_all }}" + + - name: Run test + when: ansible_play_hosts_all | length > 1 + tags: tests::verify + vars: + # Run the role with the play variable generated above + ha_cluster_node_options: "{{ __test_node_options }}" + block: + - name: Set up test environment + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup.yml + + # Generate an "inventory" variable for each node. + # Number of SBD devices is the same for all nodes. This is expected to + # be overriden with the play variable in the role. + - name: Set SBD devices variable + set_fact: + ha_cluster: + sbd_devices: + - /tmp/dev/sdx1 + + - name: Print set SBD devices (inventory) + debug: + var: ha_cluster.sbd_devices + + - name: Print set SBD devices (play) + debug: + var: ha_cluster_node_options + + - name: Run the role and check for errors + block: + - name: Run the role + include_role: + name: linux-system-roles.ha_cluster + rescue: + - name: Check errors + assert: + that: ansible_failed_result.msg == expected_msg + run_once: true # noqa: run_once[task] + vars: + expected_msg: >- + All nodes must have the same number of SBD devices specified + always: + - name: Unset SBD devices variable + set_fact: + ha_cluster: + __test_node_options: + + - name: Message + debug: + msg: This test needs two or more nodes + when: ansible_play_hosts_all | length <= 1 diff --git a/tests/tests_sbd_check_devices_count.yml b/tests/tests_sbd_check_devices_count_inventory.yml similarity index 80% rename from tests/tests_sbd_check_devices_count.yml rename to tests/tests_sbd_check_devices_count_inventory.yml index 8a3cdb1c..4db94de0 100644 --- a/tests/tests_sbd_check_devices_count.yml +++ b/tests/tests_sbd_check_devices_count_inventory.yml @@ -1,6 +1,10 @@ # SPDX-License-Identifier: MIT --- -- name: Ensure all nodes have the same number of SBD devices +# All nodes must have the same number of SBD devices. The role is supposed to +# exit with an error if that's not the case. This test verifies, that the check +# works, when SBD devices are defined in 'ha_cluster' variable (inventory +# style). +- name: Test a check SBD devices count matches for all nodes (inventory) hosts: all vars_files: vars/main.yml vars: @@ -16,6 +20,7 @@ name: linux-system-roles.ha_cluster tasks_from: test_setup.yml + # Generate an "inventory" variable for each node - name: Set SBD devices variable set_fact: ha_cluster: diff --git a/tests/tests_sbd_check_devices_count_play.yml b/tests/tests_sbd_check_devices_count_play.yml new file mode 100644 index 00000000..9a19f113 --- /dev/null +++ b/tests/tests_sbd_check_devices_count_play.yml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: MIT +--- +# All nodes must have the same number of SBD devices. The role is supposed to +# exit with an error if that's not the case. This test verifies, that the check +# works, when SBD devices are defined in 'ha_cluster_node_options' variable +# (playbook style). +- name: Test a check SBD devices count matches for all nodes (play) + hosts: all + vars_files: vars/main.yml + vars: + ha_cluster_sbd_enabled: true + + tasks: + # Generate a play variable. Play variables are equal for all nodes. + # This variable is expected to hold data for all nodes. + # Number of SBD devices is different for each node. + - name: Generate SBD devices variable + set_fact: + __test_node_options: >- + {{ + (__test_node_options | d([])) + [{ + 'node_name': item, + 'sbd_devices': ( + range(1, ansible_play_hosts_all.index(item) + 2) + | product(['/tmp/dev/sdx']) | map('reverse') | map('join') + | list) + }] + }} + loop: "{{ ansible_play_hosts_all }}" + + - name: Run test + when: ansible_play_hosts_all | length > 1 + tags: tests::verify + vars: + # Run the role with the play variable generated above + ha_cluster_node_options: "{{ __test_node_options }}" + block: + - name: Set up test environment + include_role: + name: linux-system-roles.ha_cluster + tasks_from: test_setup.yml + + - name: Print set SBD devices + debug: + var: ha_cluster_node_options + + - name: Run the role and check for errors + block: + - name: Run the role + include_role: + name: linux-system-roles.ha_cluster + rescue: + - name: Check errors + assert: + that: ansible_failed_result.msg == expected_msg + run_once: true # noqa: run_once[task] + vars: + expected_msg: >- + All nodes must have the same number of SBD devices specified + always: + - name: Unset SBD devices variable + set_fact: + __test_node_options: + + - name: Message + debug: + msg: This test needs two or more nodes + when: ansible_play_hosts_all | length <= 1