ovirtlago: reconfigure timeouts in 'lago ovirt start'

1. Increased default timeout for waiting until the VMs connected
to the Engine report 'up' status to 8 minutes.
2. Added '--vms-timeout' parameter to allow configuring that timeout.
3. Increased 'ovirt-engine'/'vdsmd' service timeout to 3 minutes.
4. Added default timeouts in all relevant functions headers, and set
the timeouts explicitly in 'do_ovirt_start' for visibility.

Signed-off-by: Nadav Goldin <ngoldin@redhat.com>

ovirtlago/cmd.py

@@ -198,10 +198,18 @@ def do_ovirt_stop_vms(prefix, **kwargs):
 
 
 @cli_plugin(help='Start all VMs that are down')
+@cli_plugin_add_argument(
+    '--vms-timeout',
+    help=('Time to wait until the Engine reports the VMs are up.'),
+    dest='vms_timeout',
+    default=8 * 60,
+    type=int,
+    action='store',
+)
 @in_ovirt_prefix
 @with_logging
-def do_ovirt_start_vms(prefix, **kwargs):
-    prefix.virt_env.engine_vm().start_all_vms()
+def do_ovirt_start_vms(prefix, vms_timeout, **kwargs):
+    prefix.virt_env.engine_vm().start_all_vms(timeout=vms_timeout)
 
 
 @cli_plugin(help='Print oVirt setup status')
@@ -223,22 +231,30 @@ def do_ovirt_status(prefix, **kwargs):
     dest='with_vms',
     action='store_true',
 )
+@cli_plugin_add_argument(
+    '--vms-timeout',
+    help=('Time to wait until the Engine reports the VMs are up.'),
+    dest='vms_timeout',
+    default=8 * 60,
+    type=int,
+    action='store',
+)
 @in_ovirt_prefix
 @with_logging
-def do_ovirt_start(prefix, with_vms, **kwargs):
+def do_ovirt_start(prefix, with_vms, vms_timeout, **kwargs):
     with LogTask('Starting oVirt environment'):
         prefix.start()
         with LogTask('Waiting for ovirt-engine status'):
-            prefix.virt_env.assert_engine_alive()
+            prefix.virt_env.assert_engine_alive(timeout=3 * 60)
         with LogTask('Waiting for vdsmd status'):
-            prefix.virt_env.assert_vdsm_alive()
+            prefix.virt_env.assert_vdsm_alive(timeout=3 * 60)
         with LogTask('Activating Engine Hosts'):
-            prefix.virt_env.engine_vm().start_all_hosts()
+            prefix.virt_env.engine_vm().start_all_hosts(timeout=5 * 60)
         if with_vms:
             with LogTask('Waiting for Storage domains to be in active mode'):
-                prefix.virt_env.engine_vm().check_sds_status()
+                prefix.virt_env.engine_vm().check_sds_status(timeout=5 * 60)
             with LogTask('Starting Engine VMs'):
-                prefix.virt_env.engine_vm().start_all_vms()
+                prefix.virt_env.engine_vm().start_all_vms(timeout=vms_timeout)
 
 
 @cli_plugin(

ovirtlago/virt.py

@@ -340,7 +340,7 @@ def _search_vms(self, api, query):
         return [vm.id for vm in vms_service.list(search=query)]
 
     @require_sdk(version='4')
-    def start_all_vms(self):
+    def start_all_vms(self, timeout=8 * 60):
         api = self.get_api_v4(check=True)
         vms_service = api.system_service().vms_service()
         ids = self._search_vms(api, query='status=down')
@@ -355,11 +355,11 @@ def _vm_is_up(id):
 
         for id in ids:
             testlib.assert_true_within(
-                partial(_vm_is_up, id=id), timeout=5 * 60
+                partial(_vm_is_up, id=id), timeout=timeout
             )
 
     @require_sdk(version='4')
-    def stop_all_vms(self):
+    def stop_all_vms(self, timeout=5 * 60):
         api = self.get_api_v4(check=True)
         vms_service = api.system_service().vms_service()
         ids = self._search_vms(api, query='status=up')
@@ -374,11 +374,11 @@ def _vm_is_down(id):
 
         for id in ids:
             testlib.assert_true_within(
-                partial(_vm_is_down, id=id), timeout=5 * 60
+                partial(_vm_is_down, id=id), timeout=timeout
             )
 
     @require_sdk(version='4')
-    def stop_all_hosts(self):
+    def stop_all_hosts(self, timeout=5 * 60):
         api = self.get_api_v4(check=True)
         hosts_service = api.system_service().hosts_service()
         hosts = hosts_service.list(search='status=up')
@@ -406,10 +406,10 @@ def _host_is_maint():
                     )
 
             for h in hosts:
-                testlib.assert_true_within(_host_is_maint, timeout=5 * 60)
+                testlib.assert_true_within(_host_is_maint, timeout=timeout)
 
     @require_sdk(version='4')
-    def start_all_hosts(self):
+    def start_all_hosts(self, timeout=5 * 60):
         api = self.get_api_v4(check=True)
         hosts_service = api.system_service().hosts_service()
         hosts = hosts_service.list(search='status=maintenance')
@@ -436,11 +436,11 @@ def _host_is_up(host):
 
             for host in hosts:
                 testlib.assert_true_within(
-                    partial(_host_is_up, host), timeout=5 * 60
+                    partial(_host_is_up, host), timeout=timeout
                 )
 
     @require_sdk(version='4')
-    def check_sds_status(self, status=None):
+    def check_sds_status(self, status=None, timeout=5 * 60):
         # the default status cannot be used in the function header, because
         # the v4 sdk might not be available.
         if status is None:
@@ -455,7 +455,7 @@ def _sds_state(dc_id):
                 return all(sd.status == status for sd in sds.list())
 
             testlib.assert_true_within(
-                partial(_sds_state, dc_id=dc.id), timeout=5 * 60
+                partial(_sds_state, dc_id=dc.id), timeout=timeout
             )
 
     @require_sdk(version='4')