Add a test for bluechictl monitor <node> <service>

Adds a test if a restarted service on a node> is properly monitored
by bluechictl monitor command.

Signed-off-by: Martin Perina <mperina@redhat.com>

tests/tests/tier0/bluechi-monitor-service/main.fmf

@@ -0,0 +1,2 @@
+summary: Test if a restarted service is properly monitored by bluechictl monitor
+id: 9d347466-3cf8-462a-b71e-9d4ec9d391ec

tests/tests/tier0/bluechi-monitor-service/python/monitor.py

@@ -0,0 +1,121 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import re
+import subprocess
+import threading
+import time
+import unittest
+
+from bluechi.api import Node
+
+node_name_foo = "node-foo"
+
+service_simple = "simple.service"
+
+log_file = "/tmp/bluechictl-output.txt"
+
+
+class FileFollower:
+    def __init__(self, file_desc):
+        self.pos = 0
+        self.file_desc = file_desc
+
+    def seek(self):
+        self.file_desc.seek(self.pos)
+
+    def new_lines(self):
+        self.seek()
+        return '\n' in self.file_desc.read()
+
+    def close(self):
+        if self.file_desc:
+            self.file_desc.close()
+
+    def __iter__(self):
+        while self.new_lines():
+            self.seek()
+            line = self.file_desc.read().split('\n')[0]
+            yield line
+
+            self.pos += len(line) + 1
+
+
+class TestMonitorSpecificNodeAndUnit(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.bluechictl_proc = None
+        self.state_pat = re.compile(r"\s*ActiveState: (?P<active_state>[\S]+)\s*")
+        self.bluechictl_out = None
+
+        self.inactive = False
+        self.active = True
+
+    def timeout_guard(self):
+        time.sleep(10)
+        print(f"Loop timeout - service '{service_simple}' on node '{node_name_foo}' "
+              f"was not successfully restarted on time")
+        self.bluechictl_proc.terminate()
+        self.bluechictl_proc.wait()
+
+    def process_events(self):
+        out_file = None
+        try:
+            out_file = open(log_file, mode='w')
+
+            self.bluechictl_proc = subprocess.Popen(
+                ["/usr/bin/bluechictl", "monitor", f"{node_name_foo}", f"{service_simple}"],
+                stdout=out_file,
+                bufsize=1)
+
+            self.bluechictl_out = FileFollower(open(log_file, mode='r'))
+            events_received = False
+            while self.bluechictl_proc.poll() is None and not events_received:
+                for line in self.bluechictl_out:
+                    print(f"Evaluating line '{line}'")
+                    match = self.state_pat.match(line)
+                    if not match:
+                        print(f"Ignoring line '{line}'")
+                        continue
+                    active_state = match.group("active_state")
+                    if active_state == "inactive" and self.active and not self.inactive:
+                        # Active -> Inactive
+                        print(f"Service '{service_simple}' on node '{node_name_foo}' was stopped successfully")
+                        self.active = False
+                        self.inactive = True
+                    elif active_state == "active" and self.inactive:
+                        # Inactive -> Active
+                        print(f"Service '{service_simple}' on node '{node_name_foo}' was started successfully")
+                        self.active = True
+                        events_received = True
+                        break
+
+                # Wait for the new output from bluechictl monitor
+                time.sleep(0.5)
+        finally:
+            if out_file:
+                out_file.close()
+            if self.bluechictl_proc:
+                self.bluechictl_proc.terminate()
+                self.bluechictl_proc.wait()
+                self.bluechictl_proc = None
+            if self.bluechictl_out:
+                self.bluechictl_out.close()
+
+    def test_monitor_specific_node_and_unit(self):
+        t = threading.Thread(target=self.process_events)
+        # mark the failsafe thread as daemon so it stops when the main process stops
+        failsafe_thread = threading.Thread(target=self.timeout_guard, daemon=True)
+        t.start()
+        failsafe_thread.start()
+
+        node_foo = Node(node_name_foo)
+        assert node_foo.restart_unit(service_simple, "replace") != ""
+
+        t.join()
+
+        assert self.inactive
+        assert self.active
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/tests/tier0/bluechi-monitor-service/systemd/simple.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Run a service for 15 seconds
+
+[Service]
+Type=simple
+ExecStart=/bin/true
+RemainAfterExit=yes

tests/tests/tier0/bluechi-monitor-service/test_bluechi_monitor_service.py

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import os
+from typing import Dict
+
+from bluechi_test.test import BluechiTest
+from bluechi_test.machine import BluechiControllerMachine, BluechiAgentMachine
+from bluechi_test.config import BluechiControllerConfig, BluechiAgentConfig
+
+node_name_foo = "node-foo"
+service_simple = "simple.service"
+
+
+def exec(ctrl: BluechiControllerMachine, nodes: Dict[str, BluechiAgentMachine]):
+    nodes[node_name_foo].copy_systemd_service(
+        service_simple, "systemd", os.path.join("/", "etc", "systemd", "system"))
+    assert nodes[node_name_foo].wait_for_unit_state_to_be(service_simple, "inactive")
+
+    ctrl.bluechictl.start_unit(node_name_foo, service_simple)
+    nodes[node_name_foo].wait_for_unit_state_to_be(service_simple, "active")
+
+    result, output = ctrl.run_python(os.path.join("python", "monitor.py"))
+    if result != 0:
+        raise Exception(output)
+
+
+def test_bluechi_monitor_service(
+        bluechi_test: BluechiTest,
+        bluechi_ctrl_default_config: BluechiControllerConfig,
+        bluechi_node_default_config: BluechiAgentConfig):
+
+    bluechi_node_default_config.node_name = node_name_foo
+    bluechi_ctrl_default_config.allowed_node_names = [bluechi_node_default_config.node_name]
+
+    bluechi_test.set_bluechi_controller_config(bluechi_ctrl_default_config)
+    bluechi_test.add_bluechi_agent_config(bluechi_node_default_config)
+
+    bluechi_test.run(exec)