Retry update_request with back-off factor on failure

When the API is momentarily unavailable, the request is not patched when the
failed_request_callback function is called in case of a worker error
and the request gets stuck.

Refers to CLOUDDST-17935

iib/workers/api_utils.py

@@ -6,13 +6,22 @@
 import requests
 from requests.packages.urllib3.util.retry import Retry
 import requests_kerberos
+from tenacity import (
+    before_sleep_log,
+    retry,
+    retry_if_exception_type,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 from iib.exceptions import IIBError
+from iib.workers.config import get_worker_config
 from iib.workers.tasks.iib_static_types import UpdateRequestPayload
 import time
 from iib.common.tracing import instrument_tracing
 
 log = logging.getLogger(__name__)
+config = get_worker_config()
 
 
 def get_requests_session(auth: bool = False) -> requests.Session:
@@ -46,10 +55,6 @@ def get_request(request_id: int) -> Dict[str, Any]:
     :rtype: dict
     :raises IIBError: if the HTTP request fails
     """
-    # Prevent a circular import
-    from iib.workers.config import get_worker_config
-
-    config = get_worker_config()
     request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}'
     log.info('Getting the request %d', request_id)
 
@@ -120,6 +125,13 @@ def set_omps_operator_version(
     return update_request(request_id, payload, exc_msg=exc_msg)
 
 
+@retry(
+    before_sleep=before_sleep_log(log, logging.WARNING),
+    reraise=True,
+    retry=retry_if_exception_type(IIBError),
+    stop=stop_after_attempt(config.iib_total_attempts),
+    wait=wait_exponential(config.iib_retry_multiplier),
+)
 def update_request(
     request_id: int,
     payload: UpdateRequestPayload,
@@ -137,9 +149,6 @@ def update_request(
     """
     # Prevent a circular import
     start_time = time.time()
-    from iib.workers.config import get_worker_config
-
-    config = get_worker_config()
     request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}'
     log.info('Patching the request %d with %r', request_id, payload)
 

tests/test_workers/test_api_utils.py

@@ -6,6 +6,9 @@
 
 from iib.exceptions import IIBError
 from iib.workers import api_utils
+from iib.workers.config import get_worker_config
+
+config = get_worker_config()
 
 
 @mock.patch('iib.workers.api_utils.requests_session')
@@ -76,6 +79,7 @@ def test_update_request_connection_failed(mock_session):
 
     with pytest.raises(IIBError, match='The connection failed.+'):
         api_utils.update_request(3, {'index_image': 'index-image:latest'})
+        assert mock_session.patch.call_count == config.iib_total_attempts
 
 
 @pytest.mark.parametrize(