Support moving state to persistent storage

.github/workflows/stackhpc.yml

@@ -100,7 +100,16 @@ jobs:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
           TEST_USER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
-
+
+      - name: Run MPI-based tests
+        run: |
+          . venv/bin/activate
+          . environments/${{ matrix.cloud }}/activate
+          ansible-playbook -vv ansible/adhoc/hpctests.yml
+        env:
+          ANSIBLE_FORCE_COLOR: True
+          OS_CLOUD: openstack
+
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |
           . venv/bin/activate
@@ -154,11 +163,11 @@ jobs:
           OS_CLOUD: openstack
           ANSIBLE_FORCE_COLOR: True
 
-      - name: Run MPI-based tests
+      - name: Check sacct state survived reimage
         run: |
           . venv/bin/activate
           . environments/${{ matrix.cloud }}/activate
-          ansible-playbook -vv ansible/adhoc/hpctests.yml
+          ansible-playbook -vv ansible/ci/check_sacct_hpctests.yml
         env:
           ANSIBLE_FORCE_COLOR: True
           OS_CLOUD: openstack

ansible/.gitignore

@@ -26,3 +26,7 @@ roles/*
 !roles/slurm_exporter/**
 !roles/firewalld/
 !roles/firewalld/**
+!roles/mysql/
+!roles/mysql/**
+!roles/systemd/
+!roles/systemd/**

ansible/bootstrap.yml

@@ -16,16 +16,36 @@
 - hosts: cluster
   gather_facts: false
   tasks:
+    - name: Add groups
+      ansible.builtin.group: "{{ item.group }}"
+      loop: "{{ appliances_local_users }}"
+      when:
+        - item.enable | default(true) | bool
+        - "'group' in item"
+      become_method: "sudo"
+      # Need to change working directory otherwise we try to switch back to non-existent directory.
+      become_flags: '-i'
+      become: true
     - name: Add users
-      ansible.builtin.user: "{{ item }}"
-      with_items: "{{ appliances_local_users }}"
+      ansible.builtin.user: "{{ item.user }}"
+      loop: "{{ appliances_local_users }}"
+      when: item.enable | default(true) | bool
       become_method: "sudo"
       # Need to change working directory otherwise we try to switch back to non-existent directory.
       become_flags: '-i'
       become: true
     - name: Reset ssh connection to allow user changes to affect ansible_user
       meta: reset_connection
 
+- hosts: systemd
+  become: yes
+  gather_facts: false
+  tags: systemd
+  tasks:
+    - name: Make systemd unit modifications
+      import_role:
+        name: systemd
+
 - hosts: selinux
   gather_facts: false
   become: yes

ansible/ci/check_sacct_hpctests.yml

@@ -0,0 +1,27 @@
+- hosts: control
+  gather_facts: false
+  become: true
+  vars:
+    sacct_stdout_expected: |- # based on CI running hpctests as the first job - NB note no trailing newline
+      JobID,JobName,State
+      2,pingpong.sh,COMPLETED
+      3,pingmatrix.sh,COMPLETED
+      4,hpl-build-linux64.sh,COMPLETED
+      5_0,hpl-solo.sh,COMPLETED
+      5_1,hpl-solo.sh,COMPLETED
+  tasks:
+    - name: Get info for ended jobs
+      shell:
+        cmd: sacct --format=jobid,jobname,state --allocations --parsable2 --delimiter=, --starttime=now-1days --endtime=now
+        # by default start/end time is midnight/now which is not robust
+      changed_when: false
+      register: sacct
+    - name: Check info for ended jobs
+      assert:
+        that: sacct.stdout == sacct_stdout_expected
+        fail_msg: |
+          Expected:
+          --{{ sacct_stdout_expected }}--
+          Got:
+          --{{ sacct.stdout }}--
+        success_msg: sacct shows hpctests jobs as first and only jobs

ansible/filter_plugins/utils.py

@@ -4,6 +4,7 @@
 # Apache 2 License
 
 from ansible.errors import AnsibleError, AnsibleFilterError
+from ansible.utils.display import Display
 from collections import defaultdict
 import jinja2
 from ansible.module_utils.six import string_types
@@ -36,10 +37,15 @@ def exists(fpath):
 class FilterModule(object):
     ''' Ansible core jinja2 filters '''
 
+    def warn(self, message, **kwargs):
+        Display().warning(message)
+        return message
+
     def filters(self):
         return {
             # jinja2 overrides
             'readfile': readfile,
             'prometheus_node_exporter_targets': prometheus_node_exporter_targets,
-            'exists': exists
-        }
+            'exists': exists,
+            'warn': self.warn
+        }

ansible/roles/block_devices/README.md

@@ -1,17 +1,19 @@
 block_devices
 =============
 
-Manage filesystems on block devices, including creating partitions, creating filesystems and mounting filesystems.
+Manage filesystems on block devices (such as OpenStack volumes), including creating partitions, creating filesystems and mounting filesystems.
 
 This is a convenience wrapper around the ansible modules:
 - community.general.parted
 - community.general.filesystem
 - ansible.buildin.file
 - ansible.posix.mount
 
-It includes logic to handle OpenStack-provided volumes appropriately both for appliance instances and the Packer build VM.
+To avoid issues with device names changing after e.g. reboots, devices are identified by serial number and mounted by filesystem UUID.
 
-To avoid issues with device names changing after e.g. reboots, devices are identified by serial number and mounted by filesystem UUID. 
+**NB:** This role is ignored[^1] during Packer builds as block devices will not be attached to the Packer build VMs. This role is therefore deprecated and it is suggested that `cloud-init` is used instead. See e.g. `environments/skeleton/{{cookiecutter.environment}}/terraform/control.userdata.tpl`.
+
+[^1]: See `environments/common/inventory/group_vars/builder/defaults.yml`
 
 Requirements
 ------------

ansible/roles/block_devices/tasks/main.yml

@@ -1,3 +1,8 @@
+- name: Warn role is deprecated
+  debug:
+    msg: "{{ 'Role block_devices is deprecated, see ansible/roles/block_devices/README.md' | warn }}"
+  when: block_devices_configurations | length > 0
+
 - name: Enumerate block device paths by serial number
   block_devices:
   register: _block_devices

ansible/roles/mysql/README.md

@@ -0,0 +1,52 @@
+mysql
+=====
+
+Deploy containerised `mysql` server using Podman.
+
+
+Requirements
+------------
+
+None.
+
+Role Variables
+--------------
+
+- `mysql_root_password`: Required str. Password to set for `root` mysql user. **NB** This cannot be changed by this role once mysql server has initialised.
+- `mysql_tag`: Optional str. Tag for version of `mysql` container image to use. Default `8.0.30`.
+- `mysql_systemd_service_enabled`: Optional bool. Whether `mysql` service starts on boot. Default `yes`.
+- `mysql_state`: Optional str. As per `ansible.builtin.systemd:state`. Default is `started` or `restarted` as required.
+- `mysql_podman_user`: Optional str. User running `podman`. Default `{{ ansible_user }}`.
+- `mysql_datadir`: Optional str. Path to data directory on the host to store databases etc. Default `/var/lib/mysql`. Note all path components will be created and user set appropriately if this does not exist.
+- `mysql_host`: Optional str. Address of host. Default `{{ inventory_hostname }}`.
+- `mysql_users`: Optional list of dicts defining users as per `community.mysql.mysql_user`. Default `[]`.
+- `mysql_databases`: Optional list of dicts defining databases as per `community.mysql.mysql_db`. Default `[]`.
+
+Dependencies
+------------
+
+None.
+
+Example Playbook
+----------------
+
+```yaml
+- name: Setup DB
+  hosts: mysql
+  become: true
+  tags:
+    - mysql
+  tasks:
+    - include_role:
+        name:  mysql
+```
+
+License
+-------
+
+Apache v2
+
+Author Information
+------------------
+
+Steve Brasier steveb@stackhpc.com

ansible/roles/mysql/defaults/main.yml

@@ -0,0 +1,11 @@
+# required:
+# mysql_root_password: # TODO: make it possible to CHANGE root password
+
+mysql_tag: 8.0.30
+mysql_systemd_service_enabled: yes
+#mysql_state: # default is started or restarted as required
+mysql_podman_user: "{{ ansible_user }}"
+mysql_datadir: /var/lib/mysql
+mysql_mysqld_options: [] # list of str options to mysqld, see `run -it --rm mysql:tag --verbose --help`
+mysql_users: [] # list of dicts for community.mysql.mysql_user
+mysql_databases: [] # list of dicts for community.mysql.mysql_db

ansible/roles/mysql/tasks/configure.yml

@@ -0,0 +1,37 @@
+- name: Create environment file for mysql server root password
+  # NB: This doesn't trigger a restart on changes as it will be ignored once mysql is initialised
+  copy:
+    dest: /etc/sysconfig/mysqld
+    content: |
+      MYSQL_INITIAL_ROOT_PASSWORD='{{ mysql_root_password }}'
+    owner: root
+    group: root
+    mode: u=rw,go=
+
+- name: Ensure mysql service state
+  systemd:
+    name: mysql
+    state: "{{ mysql_state | default('restarted' if _mysql_unitfile.changed else 'started') }}"
+    enabled: "{{ mysql_systemd_service_enabled }}"
+    daemon_reload: "{{ _mysql_unitfile.changed }}"
+
+- block:
+  - name: Wait for mysql to initialise
+    # NB: It is not sufficent to wait_for the port
+    community.mysql.mysql_info:
+      login_user: root
+      login_password: "{{ mysql_root_password }}"
+    # no_log: true # TODO: FIXME
+    register: _mysql_info
+    until: "'version' in _mysql_info"
+    retries: 60
+    delay: 2
+
+  - name: Ensure mysql databases created
+    community.mysql.mysql_db: "{{ item }}"
+    loop: "{{ mysql_databases}}"
+
+  - name: Ensure mysql users present
+    community.mysql.mysql_user: "{{ item }}"
+    loop: "{{ mysql_users }}"
+  when: "mysql_state | default('unspecified') != 'stopped'"

ansible/roles/mysql/tasks/install.yml

@@ -0,0 +1,10 @@
+- name: Install python mysql client
+  pip:
+    name: pymysql
+    state: present
+
+- name: Create systemd mysql container unit file
+  template:
+    dest: /etc/systemd/system/mysql.service
+    src: mysql.service.j2
+  register: _mysql_unitfile

ansible/roles/mysql/tasks/main.yml

@@ -0,0 +1,2 @@
+- import_tasks: install.yml
+- import_tasks: configure.yml

ansible/roles/mysql/templates/mysql.service.j2

@@ -0,0 +1,41 @@
+# mysql.service
+
+[Unit]
+Description=Podman container mysql.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+RequiresMountsFor={{ mysql_datadir }} /etc/sysconfig/mysqld
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=always
+EnvironmentFile=/etc/sysconfig/mysqld
+# The above EnvironmentFile must define MYSQL_INITIAL_ROOT_PASSWORD
+ExecStartPre=+install -d -o {{ mysql_podman_user }} -g {{ mysql_podman_user }} -Z container_file_t {{ mysql_datadir }}
+ExecStart=/usr/bin/podman run \
+    --network slirp4netns:cidr={{ podman_cidr }} \
+    --sdnotify=conmon --cgroups=no-conmon \
+    --detach --replace --name mysql --restart=no \
+    --user mysql \
+    --volume {{ mysql_datadir }}:/var/lib/mysql:U \
+    --publish 3306:3306 \
+    -e MYSQL_ROOT_PASSWORD=${MYSQL_INITIAL_ROOT_PASSWORD} \
+    mysql:{{ mysql_tag }}{%- for opt in mysql_mysqld_options %} \
+    --{{ opt }}{% endfor %}
+
+ExecStop=/usr/bin/podman stop --ignore mysql -t 10
+# note for some reason this returns status=143 which makes systemd show the unit as failed, not stopped
+ExecStopPost=/usr/bin/podman rm --ignore -f mysql
+SuccessExitStatus=143 SIGTERM
+KillMode=none
+Type=notify
+NotifyAccess=all
+LimitNOFILE=65536
+LimitMEMLOCK=infinity
+User={{ mysql_podman_user }}
+Group={{ mysql_podman_user }}
+TimeoutStartSec=180
+
+[Install]
+WantedBy=multi-user.target default.target

ansible/roles/opendistro/defaults/main.yml

@@ -3,3 +3,4 @@
 #opendistro_internal_users_path:
 
 opendistro_podman_user: "{{ ansible_user }}"
+opendistro_data_path: "/usr/share/elasticsearch/data"  # path to host data directory

ansible/roles/opendistro/templates/opendistro.service.j2

@@ -9,7 +9,21 @@ After=network-online.target
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
-ExecStart=/usr/bin/podman run --network slirp4netns:cidr={{ podman_cidr }} --sdnotify=conmon --cgroups=no-conmon -d --replace --name opendistro --restart=no --user elasticsearch --ulimit memlock=-1:-1 --ulimit nofile=65536:65536 --volume opendistro:/usr/share/elasticsearch/data --volume /etc/elastic/internal_users.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/internal_users.yml:ro --env node.name=opendistro --env discovery.type=single-node --env bootstrap.memory_lock=true --env "ES_JAVA_OPTS=-Xms512m -Xmx512m" --publish 9200:9200 amazon/opendistro-for-elasticsearch:1.12.0
+ExecStartPre=+install -d -o {{ opendistro_podman_user }} -g {{ opendistro_podman_user }} -Z container_file_t {{ opendistro_data_path }}
+ExecStart=/usr/bin/podman run \
+    --network slirp4netns:cidr={{ podman_cidr }} \
+    --sdnotify=conmon --cgroups=no-conmon \
+    --detach --replace --name opendistro --restart=no \
+    --user elasticsearch \
+    --ulimit memlock=-1:-1 --ulimit nofile=65536:65536 \
+    --volume {{ opendistro_data_path }}:/usr/share/elasticsearch/data:U \
+    --volume /etc/elastic/internal_users.yml:/usr/share/elasticsearch/plugins/opendistro_security/securityconfig/internal_users.yml:ro \
+    --env node.name=opendistro \
+    --env discovery.type=single-node \
+    --env bootstrap.memory_lock=true \
+    --env "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
+    --publish 9200:9200 \
+    amazon/opendistro-for-elasticsearch:1.12.0
 ExecStop=/usr/bin/podman stop --ignore opendistro -t 10
 # note for some reason this returns status=143 which makes systemd show the unit as failed, not stopped
 ExecStopPost=/usr/bin/podman rm --ignore -f opendistro

ansible/roles/podman/tasks/config.yml

@@ -13,6 +13,12 @@
     dest: /etc/security/limits.d/custom.conf
   become: true
 
+- name: Up default keys permitted
+  ansible.posix.sysctl:
+    name: kernel.keys.maxkeys # /proc/sys/kernel/keys/maxkeys
+    value: 50000
+  become: true
+
 - name: reset ssh connection to allow user changes to affect 'current login user'
   meta: reset_connection
 
@@ -60,9 +66,6 @@
   become: yes
   register: podman_tmp
 
-- debug:
-    var: podman_tmp
-
 - name: Reset podman database
   # otherwise old config overrides!
   command:

ansible/roles/systemd/README.md

@@ -0,0 +1,19 @@
+# systemd
+
+Create drop-in files for systemd services.
+
+# Role Variables
+- `systemd_dropins`: Required. A mapping where keys = systemd service name, values are a dict as follows:
+    - `group`: Required str. Inventory group this drop-in applies to.
+    - `comment`: Optional str. Comment describing reason for drop-in.
+    - `content`: Required str. Content of drop-in file.
+# systemd
+
+Create drop-in files for systemd services.
+
+# Role Variables
+- `systemd_dropins`: Required. A mapping where keys = systemd service name, values are a dict as follows:
+    - `group`: Required str. Inventory group this drop-in applies to.
+    - `comment`: Optional str. Comment describing reason for drop-in.
+    - `content`: Required str. Content of drop-in file.
+- `systemd_restart`: Optional bool. Whether to reload unit definitions and restart services. Default `false`.