Skip to content

Commit

Permalink
Merge pull request #221 from reportportal/develop
Browse files Browse the repository at this point in the history
Release
  • Loading branch information
HardNorth authored Oct 13, 2023
2 parents 34c7000 + 117ca91 commit 66a0639
Show file tree
Hide file tree
Showing 82 changed files with 6,085 additions and 2,342 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,10 +50,11 @@ jobs:
pip install tox tox-gh-actions
- name: Test with tox
timeout-minutes: 10
run: tox

- name: Upload coverage to Codecov
if: matrix.python-version == 3.7 && success()
if: matrix.python-version == 3.8 && success()
uses: codecov/codecov-action@v3
with:
files: coverage.xml
Expand Down
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,18 @@
# Changelog

## [Unreleased]
### Added
- `RP` class in `reportportal_client.client` module as common interface for all ReportPortal clients, by @HardNorth
- `reportportal_client.aio` with asynchronous clients and auxiliary classes, by @HardNorth
- Dependency on `aiohttp` and `certifi`, by @HardNorth
### Changed
- RPClient class does not use separate Thread for log processing anymore, by @HardNorth
- Use `importlib.metadata` package for distribution data extraction for Python versions starting 3.8, by @HardNorth
- `helpers.verify_value_length` function updated to truncate attribute keys also and reveal attributes were truncated, by @HardNorth
### Removed
- Dependency on `six`, by @HardNorth

## [5.4.1]
### Changed
- Unified ReportPortal product naming, by @HardNorth
- `RPClient` internal item stack implementation changed to `LifoQueue` to maintain concurrency better, by @HardNorth
Expand Down
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
include MANIFEST.in
include README.md CONTRIBUTING.rst requirements.txt
exclude test_res/*
130 changes: 113 additions & 17 deletions reportportal_client/__init__.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,124 @@
"""
Copyright (c) 2022 https://reportportal.io .
# Copyright (c) 2023 EPAM Systems
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
"""This package is the base package for ReportPortal client."""
import typing
import warnings

https://www.apache.org/licenses/LICENSE-2.0
import aenum

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
# noinspection PyProtectedMember
from reportportal_client._internal.local import current, set_current
from reportportal_client.aio.client import AsyncRPClient, BatchedRPClient, ThreadedRPClient
from reportportal_client.client import RP, RPClient, OutputType
from reportportal_client.logs import RPLogger, RPLogHandler
from reportportal_client.steps import step


class ClientType(aenum.Enum):
"""Enum of possible type of ReportPortal clients."""

SYNC = aenum.auto()
ASYNC = aenum.auto()
ASYNC_THREAD = aenum.auto()
ASYNC_BATCHED = aenum.auto()


# noinspection PyIncorrectDocstring
def create_client(
client_type: ClientType,
endpoint: str,
project: str,
*,
api_key: str = None,
**kwargs: typing.Any
) -> typing.Optional[RP]:
"""Create and ReportPortal Client based on the type and arguments provided.
:param client_type: Type of the Client to create.
:type client_type: ClientType
:param endpoint: Endpoint of the ReportPortal service.
:type endpoint: str
:param project: Project name to report to.
:type project: str
:param api_key: Authorization API key.
:type api_key: str
:param launch_uuid: A launch UUID to use instead of starting own one.
:type launch_uuid: str
:param is_skipped_an_issue: Option to mark skipped tests as not 'To Investigate' items on the server
side.
:type is_skipped_an_issue: bool
:param verify_ssl: Option to skip ssl verification.
:type verify_ssl: typing.Union[bool, str]
:param retries: Number of retry attempts to make in case of connection / server
errors.
:type retries: int
:param max_pool_size: Option to set the maximum number of connections to save the pool.
:type max_pool_size: int
:param http_timeout : A float in seconds for connect and read timeout. Use a Tuple to
specific connect and read separately.
:type http_timeout: Tuple[float, float]
:param mode: Launch mode, all Launches started by the client will be in that mode.
:type mode: str
:param launch_uuid_print: Print Launch UUID into passed TextIO or by default to stdout.
:type launch_uuid_print: bool
:param print_output: Set output stream for Launch UUID printing.
:type print_output: OutputType
:param log_batch_size: Option to set the maximum number of logs that can be processed in one
batch.
:type log_batch_size: int
:param log_batch_payload_limit: Maximum size in bytes of logs that can be processed in one batch.
:type log_batch_payload_limit: int
:param keepalive_timeout: For Async Clients only. Maximum amount of idle time in seconds before
force connection closing.
:type keepalive_timeout: int
:param task_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for a
Task processing.
:type task_timeout: float
:param shutdown_timeout: For Async Threaded and Batched Clients only. Time limit in seconds for
shutting down internal Tasks.
:type shutdown_timeout: float
:param trigger_num: For Async Batched Client only. Number of tasks which triggers Task batch
execution.
:type trigger_num: int
:param trigger_interval: For Async Batched Client only. Time limit which triggers Task batch
execution.
:type trigger_interval: float
:return: ReportPortal Client instance.
"""
if client_type is ClientType.SYNC:
return RPClient(endpoint, project, api_key=api_key, **kwargs)
if client_type is ClientType.ASYNC:
return AsyncRPClient(endpoint, project, api_key=api_key, **kwargs)
if client_type is ClientType.ASYNC_THREAD:
return ThreadedRPClient(endpoint, project, api_key=api_key, **kwargs)
if client_type is ClientType.ASYNC_BATCHED:
return BatchedRPClient(endpoint, project, api_key=api_key, **kwargs)
warnings.warn(f'Unknown ReportPortal Client type requested: {client_type}', RuntimeWarning, stacklevel=2)

from ._local import current
from .logs import RPLogger, RPLogHandler
from .client import RPClient
from .steps import step

__all__ = [
'ClientType',
'create_client',
'current',
'set_current',
'RP',
'RPClient',
'AsyncRPClient',
'BatchedRPClient',
'ThreadedRPClient',
'OutputType',
'RPLogger',
'RPLogHandler',
'RPClient',
'step',
]
14 changes: 14 additions & 0 deletions reportportal_client/_internal/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright (c) 2023 EPAM Systems
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

"""ReportPortal client internal API. No warnings before breaking changes. No backward compatibility."""
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
"""This package contains unit tests for logging."""

# Copyright (c) 2022 EPAM Systems
# Copyright (c) 2023 EPAM Systems
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
Expand Down
164 changes: 164 additions & 0 deletions reportportal_client/_internal/aio/http.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,164 @@
# Copyright (c) 2023 EPAM Systems
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License

"""This module designed to help with asynchronous HTTP request/response handling."""

import asyncio
import sys
from types import TracebackType
from typing import Coroutine, Any, Optional, Type, Callable

from aenum import Enum
from aiohttp import ClientSession, ClientResponse, ServerConnectionError, \
ClientResponseError

DEFAULT_RETRY_NUMBER: int = 5
DEFAULT_RETRY_DELAY: float = 0.005
THROTTLING_STATUSES: set = {425, 429}
RETRY_STATUSES: set = {408, 500, 502, 503, 504, 507}.union(THROTTLING_STATUSES)


class RetryClass(int, Enum):
"""Enum contains error types and their retry delay multiply factor as values."""

SERVER_ERROR = 1
CONNECTION_ERROR = 2
THROTTLING = 3


class RetryingClientSession:
"""Class uses aiohttp.ClientSession.request method and adds request retry logic."""

_client: ClientSession
__retry_number: int
__retry_delay: float

def __init__(
self,
*args,
max_retry_number: int = DEFAULT_RETRY_NUMBER,
base_retry_delay: float = DEFAULT_RETRY_DELAY,
**kwargs
):
"""Initialize an instance of the session with arguments.
To obtain the full list of arguments please see aiohttp.ClientSession.__init__() method. This class
just bypass everything to the base method, except two local arguments 'max_retry_number' and
'base_retry_delay'.
:param max_retry_number: the maximum number of the request retries if it was unsuccessful
:param base_retry_delay: base value for retry delay, determine how much time the class will wait after
an error. Real value highly depends on Retry Class and Retry attempt number,
since retries are performed in exponential delay manner
"""
self._client = ClientSession(*args, **kwargs)
self.__retry_number = max_retry_number
self.__retry_delay = base_retry_delay

async def __nothing(self):
pass

def __sleep(self, retry_num: int, retry_factor: int) -> Coroutine:
if retry_num > 0: # don't wait at the first retry attempt
delay = (((retry_factor * self.__retry_delay) * 1000) ** retry_num) / 1000
return asyncio.sleep(delay)
else:
return self.__nothing()

async def __request(
self, method: Callable, url, **kwargs: Any
) -> ClientResponse:
"""Make a request and retry if necessary.
The method retries requests depending on error class and retry number. For no-retry errors, such as
400 Bad Request it just returns result, for cases where it's reasonable to retry it does it in
exponential manner.
"""
result = None
exceptions = []
for i in range(self.__retry_number + 1): # add one for the first attempt, which is not a retry
retry_factor = None
try:
result = await method(url, **kwargs)
except Exception as exc:
exceptions.append(exc)
if isinstance(exc, ServerConnectionError) or isinstance(exc, ClientResponseError):
retry_factor = RetryClass.CONNECTION_ERROR

if not retry_factor:
raise exc

if result:
if result.ok or result.status not in RETRY_STATUSES:
return result

if result.status in THROTTLING_STATUSES:
retry_factor = RetryClass.THROTTLING
else:
retry_factor = RetryClass.SERVER_ERROR

if i + 1 < self.__retry_number:
# don't wait at the last attempt
await self.__sleep(i, retry_factor)

if exceptions:
if len(exceptions) > 1:
if sys.version_info > (3, 10):
# noinspection PyCompatibility
raise ExceptionGroup( # noqa: F821
'During retry attempts the following exceptions happened',
exceptions
)
else:
raise exceptions[-1]
else:
raise exceptions[0]
return result

def get(self, url: str, *, allow_redirects: bool = True,
**kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
"""Perform HTTP GET request."""
return self.__request(self._client.get, url, allow_redirects=allow_redirects, **kwargs)

def post(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
"""Perform HTTP POST request."""
return self.__request(self._client.post, url, data=data, **kwargs)

def put(self, url: str, *, data: Any = None, **kwargs: Any) -> Coroutine[Any, Any, ClientResponse]:
"""Perform HTTP PUT request."""
return self.__request(self._client.put, url, data=data, **kwargs)

def close(self) -> Coroutine:
"""Gracefully close internal aiohttp.ClientSession class instance."""
return self._client.close()

async def __aenter__(self) -> "RetryingClientSession":
"""Auxiliary method which controls what `async with` construction does on block enter."""
return self

async def __aexit__(
self,
exc_type: Optional[Type[BaseException]],
exc_val: Optional[BaseException],
exc_tb: Optional[TracebackType],
) -> None:
"""Auxiliary method which controls what `async with` construction does on block exit."""
await self.close()
Loading

0 comments on commit 66a0639

Please sign in to comment.