diff --git a/iib/workers/api_utils.py b/iib/workers/api_utils.py index b500df88..34532e17 100644 --- a/iib/workers/api_utils.py +++ b/iib/workers/api_utils.py @@ -6,13 +6,22 @@ import requests from requests.packages.urllib3.util.retry import Retry import requests_kerberos +from tenacity import ( + before_sleep_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) from iib.exceptions import IIBError +from iib.workers.config import get_worker_config from iib.workers.tasks.iib_static_types import UpdateRequestPayload import time from iib.common.tracing import instrument_tracing log = logging.getLogger(__name__) +config = get_worker_config() def get_requests_session(auth: bool = False) -> requests.Session: @@ -46,10 +55,6 @@ def get_request(request_id: int) -> Dict[str, Any]: :rtype: dict :raises IIBError: if the HTTP request fails """ - # Prevent a circular import - from iib.workers.config import get_worker_config - - config = get_worker_config() request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}' log.info('Getting the request %d', request_id) @@ -120,6 +125,13 @@ def set_omps_operator_version( return update_request(request_id, payload, exc_msg=exc_msg) +@retry( + before_sleep=before_sleep_log(log, logging.WARNING), + reraise=True, + retry=retry_if_exception_type(IIBError), + stop=stop_after_attempt(config.iib_total_attempts), + wait=wait_exponential(config.iib_retry_multiplier), +) def update_request( request_id: int, payload: UpdateRequestPayload, @@ -137,9 +149,6 @@ def update_request( """ # Prevent a circular import start_time = time.time() - from iib.workers.config import get_worker_config - - config = get_worker_config() request_url = f'{config.iib_api_url.rstrip("/")}/builds/{request_id}' log.info('Patching the request %d with %r', request_id, payload) diff --git a/tests/test_workers/test_api_utils.py b/tests/test_workers/test_api_utils.py index d8ab5085..9c5fd701 100644 --- a/tests/test_workers/test_api_utils.py +++ b/tests/test_workers/test_api_utils.py @@ -6,6 +6,9 @@ from iib.exceptions import IIBError from iib.workers import api_utils +from iib.workers.config import get_worker_config + +config = get_worker_config() @mock.patch('iib.workers.api_utils.requests_session') @@ -76,6 +79,7 @@ def test_update_request_connection_failed(mock_session): with pytest.raises(IIBError, match='The connection failed.+'): api_utils.update_request(3, {'index_image': 'index-image:latest'}) + assert mock_session.patch.call_count == config.iib_total_attempts @pytest.mark.parametrize(