Feature: Retry App Store Connect API requests that fail with server e…

…rrors (#249)

CHANGELOG.md

@@ -1,6 +1,9 @@
-Version 0.29.4
+Version 0.30.0
 -------------
 
+**Features**:
+- Tool `app-store-connect` can now retry App Store Connect API requests that fail with status 5xx (server error). Number of retries can be configured by command line option `--api-server-error-retries`, or respective environment variable `APP_STORE_CONNECT_API_SERVER_ERROR_RETRIES`. [PR #249](https://github.com/codemagic-ci-cd/cli-tools/pull/249)
+
 **Development**
 - Save unexpected exception information and stacktrace to `$TMPDIR/codemagic-cli-tools/exceptions/yyyy-mm-dd/`. [PR #248](https://github.com/codemagic-ci-cd/cli-tools/pull/248)
 - Save unsuccessful App Store Connect HTTP request and response information to `$TMPDIR/codemagic-cli-tools/failed-http-requests/yyyy-mm-dd/`. [PR #248](https://github.com/codemagic-ci-cd/cli-tools/pull/248)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "codemagic-cli-tools"
-version = "0.29.4"
+version = "0.30.0"
 description = "CLI tools used in Codemagic builds"
 readme = "README.md"
 authors = [

src/codemagic/__version__.py

@@ -1,5 +1,5 @@
 __title__ = 'codemagic-cli-tools'
 __description__ = 'CLI tools used in Codemagic builds'
-__version__ = '0.29.4.dev'
+__version__ = '0.30.0.dev'
 __url__ = 'https://github.com/codemagic-ci-cd/cli-tools'
 __licence__ = 'GNU General Public License v3.0'

src/codemagic/apple/app_store_connect/api_client.py

@@ -43,6 +43,7 @@ def __init__(
             private_key: str,
             log_requests: bool = False,
             unauthorized_request_retries: int = 1,
+            server_error_retries: int = 1,
             enable_jwt_cache: bool = False,
     ):
         """
@@ -61,6 +62,7 @@ def __init__(
             self.generate_auth_headers,
             log_requests=log_requests,
             unauthorized_request_retries=unauthorized_request_retries,
+            server_error_retries=server_error_retries,
             revoke_auth_info=self._jwt_manager.revoke,
         )
 

src/codemagic/apple/app_store_connect/api_session.py

@@ -14,17 +14,19 @@
 class AppStoreConnectApiSession(requests.Session):
 
     def __init__(
-            self,
-            auth_headers_factory: Callable[[], Dict[str, str]],
-            log_requests: bool = False,
-            unauthorized_request_retries: int = 1,
-            revoke_auth_info: Callable[[], None] = lambda: None,
+        self,
+        auth_headers_factory: Callable[[], Dict[str, str]],
+        log_requests: bool = False,
+        unauthorized_request_retries: int = 1,
+        server_error_retries: int = 1,
+        revoke_auth_info: Callable[[], None] = lambda: None,
     ):
         super().__init__()
         self._auth_headers_factory = auth_headers_factory
         self._revoke_auth_info = revoke_auth_info
         self._logger = log.get_logger(self.__class__, log_to_stream=log_requests)
         self._unauthorized_retries = unauthorized_request_retries
+        self._server_error_retries = server_error_retries
 
     def _log_response(self, response):
         try:
@@ -40,29 +42,57 @@ def _log_request(self, *args, **kwargs):
             body = {k: (v if 'password' not in k.lower() else '*******') for k, v in body.items()}
         self._logger.info(f'>>> {method} {url} {body}')
 
+    def _handle_unauthorized_request_response(self, attempt: int, response: requests.Response):
+        self._revoke_auth_info()
+        if attempt >= self._unauthorized_retries:
+            self._logger.info(f'Unauthorized request retries are exhausted with {attempt} attempts, stop trying')
+            raise AppStoreConnectApiError(response)
+
+        self._logger.info(f'Request failed due to authentication failure on attempt #{attempt}, try again')
+
+    def _handle_server_error_response(self, attempt: int, response: requests.Response):
+        if attempt >= self._server_error_retries:
+            self._logger.info(f'Server error retries are exhausted with {attempt} attempts, stop trying')
+            raise AppStoreConnectApiError(response)
+
+        self._logger.info(f'Request failed due to server error {response.status_code} on attempt #{attempt}, try again')
+
+    def _do_request(
+        self,
+        *request_args,
+        unauthorized_attempt: int = 1,
+        server_error_attempt: int = 1,
+        **request_kwargs,
+    ) -> requests.Response:
+        self._log_request(*request_args, **request_kwargs)
+        headers = request_kwargs.pop('headers', {})
+        headers.update(self._auth_headers_factory())
+        request_kwargs['headers'] = headers
+        response = super().request(*request_args, **request_kwargs)
+        self._log_response(response)
+
+        if response.ok:
+            return response
+
+        # Request failed, save request info and see if we can retry it
+        auditing.save_http_request_audit(response, audit_directory_name='failed-http-requests')
+
+        if response.status_code == 401:
+            self._handle_unauthorized_request_response(unauthorized_attempt, response)
+            unauthorized_attempt = unauthorized_attempt + 1
+        elif response.status_code >= 500:
+            self._handle_server_error_response(server_error_attempt, response)
+            server_error_attempt = server_error_attempt + 1
+        else:
+            # Neither authorization failure nor server error, fail immediately
+            raise AppStoreConnectApiError(response)
+
+        return self._do_request(
+            *request_args,
+            unauthorized_attempt=unauthorized_attempt,
+            server_error_attempt=server_error_attempt,
+            **request_kwargs,
+        )
+
     def request(self, *args, **kwargs) -> requests.Response:
-        for attempt in range(1, self._unauthorized_retries+1):
-            self._log_request(*args, **kwargs)
-            headers = kwargs.pop('headers', {})
-            if attempt > 1:
-                self._revoke_auth_info()
-            headers.update(self._auth_headers_factory())
-            kwargs['headers'] = headers
-            response = super().request(*args, **kwargs)
-            self._log_response(response)
-
-            if response.ok:
-                return response
-            elif response.status_code != 401:
-                # Not an authorization failure, save request audit log and fail immediately
-                auditing.save_http_request_audit(response, audit_directory_name='failed-http-requests')
-                raise AppStoreConnectApiError(response)
-            elif attempt == self._unauthorized_retries:
-                self._logger.info('Unauthorized request retries are exhausted with %d attempts, stop trying', attempt)
-                self._revoke_auth_info()
-                raise AppStoreConnectApiError(response)
-            else:
-                self._logger.info('Request failed due to authentication failure on attempt #%d, try again', attempt)
-
-        # Make mypy happy. We should never end up here.
-        raise RuntimeError('Request attempts exhausted without raising or returning')
+        return self._do_request(*args, **kwargs)

src/codemagic/tools/_app_store_connect/arguments.py

@@ -162,6 +162,15 @@ class ApiUnauthorizedRetries(cli.TypedCliArgument[int]):
         def _is_valid(cls, value: int) -> bool:
             return value > 0
 
+    class ApiServerErrorRetries(cli.TypedCliArgument[int]):
+        argument_type = int
+        environment_variable_key = 'APP_STORE_CONNECT_API_SERVER_ERROR_RETRIES'
+        default_value = 3
+
+        @classmethod
+        def _is_valid(cls, value: int) -> bool:
+            return value > 0
+
     class EarliestReleaseDate(cli.TypedCliArgument[datetime]):
         argument_type = datetime
 
@@ -393,6 +402,21 @@ class AppStoreConnectArgument(cli.Argument):
             'required': False,
         },
     )
+    SERVER_ERROR_RETRIES = cli.ArgumentProperties(
+        key='server_error_retries',
+        flags=('--api-server-error-retries',),
+        type=Types.ApiServerErrorRetries,
+        description=(
+            'Specify how many times the App Store Connect API request '
+            'should be retried in case the called request fails due to a '
+            'server error (response with status code 5xx). '
+            'In case of server error, the request is retried until '
+            'the number of retries is exhausted.'
+        ),
+        argparse_kwargs={
+            'required': False,
+        },
+    )
     DISABLE_JWT_CACHE = cli.ArgumentProperties(
         key='disable_jwt_cache',
         flags=('--disable-jwt-cache',),
@@ -537,10 +561,12 @@ class AppStoreVersionArgument(cli.Argument):
             'default': Platform.IOS,
         },
     )
-    PLATFORM_OPTIONAL = PLATFORM.duplicate(argparse_kwargs={
-        'required': False,
-        'choices': list(Platform),
-    })
+    PLATFORM_OPTIONAL = PLATFORM.duplicate(
+        argparse_kwargs={
+            'required': False,
+            'choices': list(Platform),
+        },
+    )
     RELEASE_TYPE = cli.ArgumentProperties(
         key='release_type',
         flags=('--release-type',),