Improve integration tests stability

tests/bluechi_test/container.py

@@ -116,13 +116,26 @@ def get_unit_state(self, unit_name: str) -> str:
         _, output = self.exec_run(f"systemctl is-active {unit_name}")
         return output
 
-    def wait_for_unit_state_to_be(self, unit_name: str, expected_state: str, timeout: float = 4.0) -> bool:
+    def _is_unit_in_state(self, unit_name: str, expected_state: str) -> bool:
+        latest_state = self.get_unit_state(unit_name)
+        logger.debug(f"Got state '{latest_state}' for unit {unit_name}")
+        return latest_state == expected_state
+
+    def wait_for_unit_state_to_be(
+            self,
+            unit_name: str,
+            expected_state: str,
+            timeout: float = 5.0,
+            delay: float = 0.5) -> bool:
         latest_state = ""
 
+        if self._is_unit_in_state(unit_name, expected_state):
+            return True
+
         start = time.time()
         while (time.time() - start) < timeout:
-            latest_state = self.get_unit_state(unit_name)
-            if latest_state == expected_state:
+            time.sleep(delay)
+            if self._is_unit_in_state(unit_name, expected_state):
                 return True
 
         logger.debug(f"Timeout while waiting for '{unit_name}' to reach state '{expected_state}'. \

tests/tests/tier0/bluechi-agent-resolve-fqdn/test_bluechi_agent_resolve_fqdn.py

@@ -19,20 +19,14 @@ def create_local_node_config() -> BluechiNodeConfig:
 
 
 def verify_resolving_fqdn(ctrl: BluechiControllerContainer, _: Dict[str, BluechiNodeContainer]):
-    result, output = ctrl.exec_run('systemctl is-active bluechi-controller')
-    assert result == 0
-    assert output == 'active'
-
     # create config for local bluechi-agent and adding config to controller container
     local_node_cfg = create_local_node_config()
     local_node_cfg.node_name = local_node_name
     local_node_cfg.manager_host = "localhost"
     ctrl.create_file(local_node_cfg.get_confd_dir(), local_node_cfg.file_name, local_node_cfg.serialize())
 
     ctrl.systemctl_start_and_wait("bluechi-agent", 1)
-    result, output = ctrl.exec_run('systemctl is-active bluechi-agent')
-    assert result == 0
-    assert output == 'active'
+    ctrl.wait_for_unit_state_to_be('bluechi-agent', 'active')
 
     result, output = ctrl.exec_run(f'bluechictl list-units {local_node_name}')
     assert result == 0

tests/tests/tier0/bluechi-agent-started-and-connected/test_agent_started_and_connected.py

@@ -8,11 +8,6 @@
 
 
 def foo_startup_verify(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer]):
-    result, output = ctrl.exec_run('systemctl is-active bluechi-controller')
-
-    assert result == 0
-    assert output == 'active'
-
     result, output = ctrl.exec_run('bluechictl status node-foo bluechi-agent.service')
 
     assert result == 0

tests/tests/tier0/bluechi-enable-service/test_bluechi_enable_service.py

@@ -19,6 +19,7 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     target_dir = os.path.join("/", "etc", "systemd", "system")
 
     foo.copy_systemd_service(simple_service, source_dir, target_dir)
+    assert foo.wait_for_unit_state_to_be(simple_service, "inactive")
 
     _, output = foo.exec_run(f"systemctl is-enabled {simple_service}")
     if output != "disabled":

tests/tests/tier0/bluechi-freeze-and-thaw-service/test_bluechi_freeze_and_thaw_service.py

@@ -33,6 +33,7 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     target_dir = os.path.join("/", "etc", "systemd", "system")
 
     foo.copy_systemd_service(simple_service, source_dir, target_dir)
+    assert foo.wait_for_unit_state_to_be(simple_service, "inactive")
 
     ctrl.start_unit(node_foo_name, simple_service)
     verify_service_start(foo)

tests/tests/tier0/bluechi-long-multiline-config-setting/test_long_multiline_config_setting.py

@@ -8,10 +8,7 @@
 
 
 def startup_verify(ctrl: BluechiControllerContainer, _: Dict[str, BluechiNodeContainer]):
-    result, output = ctrl.exec_run('systemctl is-active bluechi-controller')
-
-    assert result == 0
-    assert output == 'active'
+    ctrl.wait_for_unit_state_to_be('bluechi-controller', 'active')
 
 
 def test_long_multiline_config_setting(bluechi_test: BluechiTest, bluechi_ctrl_default_config: BluechiControllerConfig):

tests/tests/tier0/bluechi-service-startup/test_service_startup.py

@@ -8,10 +8,7 @@
 
 
 def startup_verify(ctrl: BluechiControllerContainer, _: Dict[str, BluechiNodeContainer]):
-    result, output = ctrl.exec_run('systemctl is-active bluechi-controller')
-
-    assert result == 0
-    assert output == 'active'
+    ctrl.wait_for_unit_state_to_be('bluechi-controller', 'active')
 
 
 def test_controller_startup(bluechi_test: BluechiTest, bluechi_ctrl_default_config: BluechiControllerConfig):

tests/tests/tier0/monitor-node-reconnect/test_monitor_node_reconnect.py

@@ -18,8 +18,7 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
         raise Exception(output)
 
     ctrl.exec_run("systemctl stop bluechi-controller")
-    _, output = ctrl.exec_run('systemctl is-active bluechi-controller')
-    assert output == 'inactive'
+    ctrl.wait_for_unit_state_to_be('bluechi-controller', 'inactive')
 
     ctrl.exec_run("systemctl start bluechi-controller")
     ctrl.wait_for_bluechi()

tests/tests/tier0/proxy-service-fails-on-non-existent-service/test_proxy_service_fails_on_non_existent_service.py

@@ -29,6 +29,7 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     target_dir = os.path.join("/", "etc", "systemd", "system")
 
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
 
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start_failed(foo)

tests/tests/tier0/proxy-service-fails-on-typo-in-file/test_proxy_service_fails_on_typo_in_file.py

@@ -36,6 +36,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start_failed(foo, bar)
 

tests/tests/tier0/proxy-service-multiple-services-multiple-nodes/test_proxy_service_multiple_services_multiple_nodes.py

@@ -29,6 +29,10 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo2.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo1.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert foo2.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     bluechi_dep_service = assemble_bluechi_dep_service_name(simple_service)
     bluechi_proxy_service = assemble_bluechi_proxy_service_name(node_bar_name, simple_service)
 

tests/tests/tier0/proxy-service-multiple-services-one-node/test_proxy_service_multiple_services_one_node.py

@@ -28,6 +28,10 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_the_second_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_the_first_service, "inactive")
+    assert foo.wait_for_unit_state_to_be(requesting_the_second_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     bluechi_dep_service = assemble_bluechi_dep_service_name(simple_service)
     bluechi_proxy_service = assemble_bluechi_proxy_service_name(node_bar_name, simple_service)
 

tests/tests/tier0/proxy-service-start/test_proxy_service_start.py

@@ -36,6 +36,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start(foo, bar)
 

tests/tests/tier0/proxy-service-stop-bluechi-dep/test_proxy_service_stop_bluechi_dep.py

@@ -46,6 +46,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start(foo, bar)
     ctrl.stop_unit(node_bar_name, assemble_bluechi_dep_service_name(simple_service))

tests/tests/tier0/proxy-service-stop-requesting-with-unneeded/test_proxy_service_stop_requesting_with_unneeded.py

@@ -46,6 +46,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start(foo, bar)
     ctrl.stop_unit(node_foo_name, requesting_service)

tests/tests/tier0/proxy-service-stop-requesting/test_proxy_service_stop_requesting.py

@@ -46,6 +46,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start(foo, bar)
     ctrl.stop_unit(node_foo_name, requesting_service)

tests/tests/tier0/proxy-service-stop-target/test_proxy_service_stop_target.py

@@ -46,6 +46,9 @@ def exec(ctrl: BluechiControllerContainer, nodes: Dict[str, BluechiNodeContainer
     foo.copy_systemd_service(requesting_service, source_dir, target_dir)
     bar.copy_systemd_service(simple_service, source_dir, target_dir)
 
+    assert foo.wait_for_unit_state_to_be(requesting_service, "inactive")
+    assert bar.wait_for_unit_state_to_be(simple_service, "inactive")
+
     ctrl.start_unit(node_foo_name, requesting_service)
     verify_proxy_start(foo, bar)
     ctrl.stop_unit(node_bar_name, simple_service)