From 231a0d4236dca854ad14fa7a6b3c3ce8e7cd6118 Mon Sep 17 00:00:00 2001 From: "reportportal.io" Date: Thu, 28 Apr 2022 12:12:53 +0000 Subject: [PATCH 01/33] Changelog update --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf4fdc5d..cd538a4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] + +## [5.2.2] ### Fixed - Issue [#182](https://github.com/reportportal/client-Python/issues/182): logger crash on attachments, by @HardNorth From 39484f2a2a191eea6a077e11668e41abd8c066dd Mon Sep 17 00:00:00 2001 From: "reportportal.io" Date: Thu, 28 Apr 2022 12:12:54 +0000 Subject: [PATCH 02/33] Version update --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 4e5d6e53..e1c4d229 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -__version__ = '5.2.2' +__version__ = '5.2.3' with open('requirements.txt') as f: requirements = f.read().splitlines() From 6dbacfe14414c75e15bdbbb0a168736fc49afc3a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 28 Apr 2022 15:15:18 +0300 Subject: [PATCH 03/33] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd538a4f..9248dc59 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## [5.2.2] ### Fixed -- Issue [#182](https://github.com/reportportal/client-Python/issues/182): logger crash on attachments, by @HardNorth +- Issue [#182](https://github.com/reportportal/client-Python/issues/182): logger crash on empty client, by @HardNorth ## [5.2.1] ### Fixed From 42733fb4a913144585ab0ee3d66daddb31161944 Mon Sep 17 00:00:00 2001 From: Dagan Sandler <1357456+dagansandler@users.noreply.github.com> Date: Mon, 20 Jun 2022 13:16:15 +0300 Subject: [PATCH 04/33] Fixes #184 Don't fail on missing 'attachment' attributre in RPLogHandler --- reportportal_client/logs/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index 0ec11232..8630bb03 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -156,7 +156,7 @@ def emit(self, record): timestamp(), msg, level=self._loglevel_map[level], - attachment=getattr(record, 'attachment'), + attachment=record.__dict__.get('attachment', None), item_id=rp_client.current_item() ) return From f7d5beab4cee57f4c4ffbd91ee4462690fb0513f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 16:55:01 +0300 Subject: [PATCH 05/33] Ability to pass client instance in `RPLogHandler` constructor --- CHANGELOG.md | 2 ++ reportportal_client/logs/__init__.py | 12 +++++++++--- tests/logs/test_rp_log_handler.py | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9248dc59..44a2d7d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ # Changelog ## [Unreleased] +### Added +- Ability to pass client instance in `RPLogHandler` constructor, by @HardNorth ## [5.2.2] ### Fixed diff --git a/reportportal_client/logs/__init__.py b/reportportal_client/logs/__init__.py index 8630bb03..6bb25705 100644 --- a/reportportal_client/logs/__init__.py +++ b/reportportal_client/logs/__init__.py @@ -98,14 +98,16 @@ class RPLogHandler(logging.Handler): def __init__(self, level=logging.NOTSET, filter_client_logs=False, endpoint=None, - ignored_record_names=tuple('reportportal_client')): + ignored_record_names=tuple('reportportal_client'), + rp_client=None): """ Initialize RPLogHandler instance. :param level: level of logging :param filter_client_logs: if True throw away logs emitted by a ReportPortal client - :param endpoint: link to send reports + :param endpoint: Report Portal endpoint URL, used to filter + out urllib3 logs, mutes Report Portal HTTP logs if set :param ignored_record_names: a tuple of record names which will be filtered out by the handler (with startswith method) """ @@ -113,6 +115,7 @@ def __init__(self, level=logging.NOTSET, filter_client_logs=False, self.filter_client_logs = filter_client_logs self.ignored_record_names = ignored_record_names self.endpoint = endpoint + self.rp_client = rp_client def filter(self, record): """Filter specific records to avoid sending those to RP. @@ -150,7 +153,10 @@ def emit(self, record): for level in self._sorted_levelnos: if level <= record.levelno: - rp_client = current() + if self.rp_client: + rp_client = self.rp_client + else: + rp_client = current() if rp_client: rp_client.log( timestamp(), diff --git a/tests/logs/test_rp_log_handler.py b/tests/logs/test_rp_log_handler.py index e981417d..2a56d948 100644 --- a/tests/logs/test_rp_log_handler.py +++ b/tests/logs/test_rp_log_handler.py @@ -107,3 +107,21 @@ def test_emit_null_client_no_error(mocked_handle): log_handler = RPLogHandler() log_handler.emit(record) + + +@mock.patch('reportportal_client.logs.logging.Logger.handle') +def test_emit_custom_client(mocked_handle): + test_message = 'test message' + RPLogger('test_logger').log(30, test_message) + record = mocked_handle.call_args[0][0] + + thread_client = mock.Mock() + direct_client = mock.Mock() + + set_current(thread_client) + + log_handler = RPLogHandler(rp_client=direct_client) + log_handler.emit(record) + + assert thread_client.log.call_count == 0 + assert direct_client.log.call_count == 1 From b9e7058ca7570dacfd66cb084dbd356f3e3756dd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 17:16:13 +0300 Subject: [PATCH 06/33] Workaround for tox issue --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index c98fba12..c94c554a 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,8 @@ envlist = py36 py37 py38 + py39 + py310 [testenv] deps = @@ -14,6 +16,7 @@ deps = delayed-assert pytest pytest-cov + importlib-metadata setenv = AGENT_NO_ANALYTICS = 1 @@ -40,3 +43,5 @@ python = 3.6: pep, py36 3.7: py37 3.8: py38 + 3.9: py39 + 3.10: py310 From 669b8eac0f62d99fa4c639fdfb1c47043af0e90c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 17:20:27 +0300 Subject: [PATCH 07/33] Revert "Workaround for tox issue" This reverts commit b9e7058ca7570dacfd66cb084dbd356f3e3756dd. --- tox.ini | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tox.ini b/tox.ini index c94c554a..c98fba12 100644 --- a/tox.ini +++ b/tox.ini @@ -7,8 +7,6 @@ envlist = py36 py37 py38 - py39 - py310 [testenv] deps = @@ -16,7 +14,6 @@ deps = delayed-assert pytest pytest-cov - importlib-metadata setenv = AGENT_NO_ANALYTICS = 1 @@ -43,5 +40,3 @@ python = 3.6: pep, py36 3.7: py37 3.8: py38 - 3.9: py39 - 3.10: py310 From ec2e198ea300e6660b23878f4a07123ed289f38e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 17:21:05 +0300 Subject: [PATCH 08/33] Workaround for setuptools issue --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 650dca55..c5d403a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine + pip install setuptools wheel twine importlib-metadata - name: Build and publish env: From 8637b2c0d02639898940f974e10fc6269989dd9a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 17:23:12 +0300 Subject: [PATCH 09/33] Revert "Workaround for setuptools issue" This reverts commit ec2e198ea300e6660b23878f4a07123ed289f38e. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5d403a3..650dca55 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine importlib-metadata + pip install setuptools wheel twine - name: Build and publish env: From dfeca833667c5615334129e0d2c12e9ed54de7be Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 17:23:40 +0300 Subject: [PATCH 10/33] Workaround for tox issue --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a1be1cac..98818567 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,5 +4,6 @@ requires = [ "setuptools>=40.0", "setuptools-scm", "wheel", + "importlib-metadata" ] build-backend = "setuptools.build_meta" From c7d92fe78f8157eb5db27c08e4972c8bba4594fd Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Tue, 21 Jun 2022 18:05:56 +0300 Subject: [PATCH 11/33] Log batch size calculation: WIP --- reportportal_client/client.py | 38 ++++++++++++++---------- reportportal_client/client.pyi | 4 ++- reportportal_client/core/log_manager.py | 25 +++++++++------- reportportal_client/core/log_manager.pyi | 4 ++- 4 files changed, 43 insertions(+), 28 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 120de724..dccacfb8 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -58,24 +58,28 @@ def __init__(self, max_pool_size=50, launch_id=None, http_timeout=(10, 10), + log_batch_payload_size=67108864, **_): """Initialize required attributes. - :param endpoint: Endpoint of the report portal service - :param project: Project name to report to - :param token: Authorization token - :param log_batch_size: Option to set the maximum number of - logs that can be processed in one batch - :param is_skipped_an_issue: Option to mark skipped tests as not - 'To Investigate' items on the server side - :param verify_ssl: Option to skip ssl verification - :param max_pool_size: Option to set the maximum number of - connections to save the pool. - :param launch_id: a launch id to use instead of starting own - one - :param http_timeout: a float in seconds for connect and read - timeout. Use a Tuple to specific connect - and read separately. + :param endpoint: Endpoint of the report portal service + :param project: Project name to report to + :param token: Authorization token + :param log_batch_size: Option to set the maximum number of + logs that can be processed in one batch + :param is_skipped_an_issue: Option to mark skipped tests as not + 'To Investigate' items on the server + side + :param verify_ssl: Option to skip ssl verification + :param max_pool_size: Option to set the maximum number of + connections to save the pool. + :param launch_id: a launch id to use instead of starting + own one + :param http_timeout: a float in seconds for connect and read + timeout. Use a Tuple to specific connect + and read separately. + :param log_batch_payload_size: maximum size in bytes of logs that can + be processed in one batch """ set_current(self) self._batch_logs = [] @@ -89,6 +93,7 @@ def __init__(self, self.is_skipped_an_issue = is_skipped_an_issue self.launch_id = launch_id self.log_batch_size = log_batch_size + self.log_batch_payload_size = log_batch_payload_size self.token = token self.verify_ssl = verify_ssl self.http_timeout = http_timeout @@ -109,7 +114,8 @@ def __init__(self, self._log_manager = LogManager( self.endpoint, self.session, self.api_v2, self.launch_id, - self.project, log_batch_size=log_batch_size) + self.project, log_batch_size=log_batch_size, + log_batch_payload_size=log_batch_payload_size) def finish_launch(self, end_time, diff --git a/reportportal_client/client.pyi b/reportportal_client/client.pyi index 4767ca7c..56ee5669 100644 --- a/reportportal_client/client.pyi +++ b/reportportal_client/client.pyi @@ -20,6 +20,7 @@ class RPClient: is_skipped_an_issue: bool = ... launch_id: Text = ... log_batch_size: int = ... + log_batch_payload_size: int = ... project: Text = ... token: Text = ... verify_ssl: bool = ... @@ -37,7 +38,8 @@ class RPClient: retries: int = ..., max_pool_size: int = ..., launch_id: Text = ..., - http_timeout: Union[float, Tuple[float, float]] = ... + http_timeout: Union[float, Tuple[float, float]] = ..., + log_batch_payload_size: int = ... ) -> None: ... def finish_launch(self, diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index ee24710d..02a4049a 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -33,18 +33,22 @@ class LogManager(object): """Manager of the log items.""" def __init__(self, rp_url, session, api_version, launch_id, project_name, - log_batch_size=20, verify_ssl=True): + log_batch_size=20, verify_ssl=True, + log_batch_payload_size=67108864): """Initialize instance attributes. - :param rp_url: Report portal URL - :param session: HTTP Session object - :param api_version: RP API version - :param launch_id: Parent launch UUID - :param project_name: RP project name - :param log_batch_size: The amount of log objects that need to be - gathered before processing - :param verify_ssl: Indicates that it is necessary to verify SSL - certificates within HTTP request + :param rp_url: Report portal URL + :param session: HTTP Session object + :param api_version: RP API version + :param launch_id: Parent launch UUID + :param project_name: RP project name + :param log_batch_size: The amount of log objects that need to + be gathered before processing + :param verify_ssl: Indicates that it is necessary to + verify SSL + certificates within HTTP request + :param log_batch_payload_size: maximum size in bytes of logs that can + be processed in one batch """ self._lock = Lock() self._logs_batch = [] @@ -53,6 +57,7 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, self.queue = queue.PriorityQueue() self.launch_id = launch_id self.log_batch_size = log_batch_size + self.log_batch_payload_size = log_batch_payload_size self.project_name = project_name self.rp_url = rp_url self.session = session diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index 7746fbc3..d310d144 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -28,6 +28,7 @@ class LogManager: rp_url: Text = ... session: Session = ... verify_ssl: bool = ... + log_batch_payload_size: int = ... def __init__(self, rp_url: Text, session: Session, @@ -35,7 +36,8 @@ class LogManager: launch_id: Text, project_name: Text, log_batch_size: int = ..., - verify_ssl: bool = ...) -> None: ... + verify_ssl: bool = ..., + log_batch_payload_size: int = ...) -> None: ... def _log_process(self, log_req: RPRequestLog) -> None: ... def _send_batch(self) -> None: ... From f38a830ca0701b7dacf30d5113f0880a204d7799 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 21 Jun 2022 21:23:33 +0300 Subject: [PATCH 12/33] Replace deprecated threading aliases --- reportportal_client/core/worker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index 7bb48f97..eb79832c 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -17,7 +17,7 @@ import logging import threading -from threading import currentThread, Thread +from threading import current_thread, Thread from aenum import auto, Enum, unique from reportportal_client.static.defines import Priority @@ -144,7 +144,7 @@ def _stop_immediately(self): may be some records still left on the queue, which won't be processed. """ self._stop_lock.acquire() - if self._thread.is_alive() and self._thread is not currentThread(): + if self._thread.is_alive() and self._thread is not current_thread(): self._thread.join(timeout=THREAD_TIMEOUT) self._thread = None self._stop_lock.notify_all() @@ -171,7 +171,7 @@ def start(self): # Already started return self._thread = Thread(target=self._monitor) - self._thread.setDaemon(True) + self._thread.daemon = True self._thread.start() def __perform_stop(self, stop_command): From 2c91baa5adc3132b89d1fd4f365e481d20bca6cc Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 14:44:39 +0300 Subject: [PATCH 13/33] Log batch size calculation: move default batch limit into constant --- reportportal_client/client.py | 4 ++-- reportportal_client/core/log_manager.py | 4 +++- reportportal_client/core/log_manager.pyi | 16 +++++++++++----- reportportal_client/core/worker.pyi | 16 ++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index dccacfb8..48a71d18 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -20,7 +20,7 @@ from requests.adapters import HTTPAdapter, Retry from ._local import set_current -from .core.log_manager import LogManager +from .core.log_manager import LogManager, MAX_LOG_BATCH_PAYLOAD_SIZE from .core.rp_requests import ( HttpRequest, ItemStartRequest, @@ -58,7 +58,7 @@ def __init__(self, max_pool_size=50, launch_id=None, http_timeout=(10, 10), - log_batch_payload_size=67108864, + log_batch_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE, **_): """Initialize required attributes. diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 02a4049a..511b31ac 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -28,13 +28,15 @@ logger = logging.getLogger(__name__) +MAX_LOG_BATCH_PAYLOAD_SIZE = 65000000 + class LogManager(object): """Manager of the log items.""" def __init__(self, rp_url, session, api_version, launch_id, project_name, log_batch_size=20, verify_ssl=True, - log_batch_payload_size=67108864): + log_batch_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE): """Initialize instance attributes. :param rp_url: Report portal URL diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index d310d144..24950a63 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -1,20 +1,20 @@ from logging import Logger -from requests import Session from threading import Lock +from typing import Dict, List, Optional, Text +from requests import Session from six.moves import queue from reportportal_client.core.rp_requests import ( - HttpRequest as HttpRequest, - RPFile as RPFile, - RPLogBatch as RPLogBatch, RPRequestLog as RPRequestLog ) from reportportal_client.core.worker import APIWorker as APIWorker -from typing import Dict, List, Optional, Text logger: Logger +MAX_LOG_BATCH_PAYLOAD_SIZE: int + + class LogManager: _lock: Lock = ... _log_endpoint: Text = ... @@ -29,6 +29,7 @@ class LogManager: session: Session = ... verify_ssl: bool = ... log_batch_payload_size: int = ... + def __init__(self, rp_url: Text, session: Session, @@ -40,13 +41,18 @@ class LogManager: log_batch_payload_size: int = ...) -> None: ... def _log_process(self, log_req: RPRequestLog) -> None: ... + def _send_batch(self) -> None: ... + def log(self, time: Text, message: Optional[Text] = ..., level: Optional[Text] = ..., attachment: Optional[Dict] = ..., item_id: Optional[Text] = ...) -> None: ... + def start(self) -> None: ... + def stop(self) -> None: ... + def stop_force(self) -> None: ... diff --git a/reportportal_client/core/worker.pyi b/reportportal_client/core/worker.pyi index 4204d8c1..d81134a6 100644 --- a/reportportal_client/core/worker.pyi +++ b/reportportal_client/core/worker.pyi @@ -12,6 +12,7 @@ from static.defines import Priority logger: Logger THREAD_TIMEOUT: int + class ControlCommand(Enum): CLEAR_QUEUE: Any = ... NOP: Any = ... @@ -20,10 +21,13 @@ class ControlCommand(Enum): STOP_IMMEDIATE: Any = ... def is_stop_cmd(self) -> bool: ... + def __lt__(self, other: Union[ControlCommand, RPRequest]) -> bool: ... + @property def priority(self) -> Priority: ... + class APIWorker: _queue: PriorityQueue = ... _thread: Optional[Thread] = ... @@ -31,15 +35,27 @@ class APIWorker: name: Text = ... def __init__(self, task_queue: PriorityQueue) -> None: ... + def _command_get(self) -> Optional[ControlCommand]: ... + def _command_process(self, cmd: Optional[ControlCommand]) -> None: ... + def _monitor(self) -> None: ... + def _request_process(self, request: Optional[RPRequest]) -> None: ... + def _stop(self) -> None: ... + def _stop_immediately(self) -> None: ... + def is_alive(self) -> bool: ... + def send(self, cmd: Union[ControlCommand, RPRequest]) -> Any: ... + def start(self) -> None: ... + def __perform_stop(self, stop_command: ControlCommand) -> None: ... + def stop(self) -> None: ... + def stop_immediate(self) -> None: ... From 1e5ea2f67552859197376925e5652cf46670d135 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 14:48:44 +0300 Subject: [PATCH 14/33] Copyrights --- reportportal_client/_local/__init__.py | 6 +++-- reportportal_client/_local/__init__.pyi | 2 +- reportportal_client/core/__init__.py | 13 ++++++++++ reportportal_client/core/log_manager.py | 27 ++++++++++---------- reportportal_client/core/log_manager.pyi | 13 ++++++++++ reportportal_client/core/rp_file.py | 13 ++++++++++ reportportal_client/core/rp_file.pyi | 13 ++++++++++ reportportal_client/core/rp_issues.py | 26 +++++++++----------- reportportal_client/core/rp_issues.pyi | 13 ++++++++++ reportportal_client/core/rp_requests.py | 27 ++++++++++---------- reportportal_client/core/rp_requests.pyi | 13 ++++++++++ reportportal_client/core/rp_responses.py | 26 ++++++++++---------- reportportal_client/core/rp_responses.pyi | 13 ++++++++++ reportportal_client/core/test_manager.py | 30 +++++++++++------------ reportportal_client/core/worker.py | 30 +++++++++++------------ reportportal_client/core/worker.pyi | 13 ++++++++++ 16 files changed, 189 insertions(+), 89 deletions(-) diff --git a/reportportal_client/_local/__init__.py b/reportportal_client/_local/__init__.py index c26d6d6e..82cc8238 100644 --- a/reportportal_client/_local/__init__.py +++ b/reportportal_client/_local/__init__.py @@ -1,4 +1,6 @@ -# Copyright (c) 2022 https://reportportal.io . +"""Report Portal client context storing and retrieving module.""" + +# Copyright (c) 2022 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -10,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License -"""Report Portal client context storing and retrieving module.""" + from threading import local __INSTANCES = local() diff --git a/reportportal_client/_local/__init__.pyi b/reportportal_client/_local/__init__.pyi index b7b14cb5..b5f2ab4e 100644 --- a/reportportal_client/_local/__init__.pyi +++ b/reportportal_client/_local/__init__.pyi @@ -1,4 +1,4 @@ -# Copyright (c) 2022 https://reportportal.io . +# Copyright (c) 2022 EPAM Systems # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/reportportal_client/core/__init__.py b/reportportal_client/core/__init__.py index b90a7e35..78e27ba6 100644 --- a/reportportal_client/core/__init__.py +++ b/reportportal_client/core/__init__.py @@ -1 +1,14 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + """This package contains core reportportal-client modules.""" diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 511b31ac..13bd6276 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -1,16 +1,17 @@ -"""This module contains management functionality for processing logs. - -Copyright (c) 2018 https://reportportal.io . -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at -https://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This module contains management functionality for processing logs.""" + +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License import logging from threading import Lock diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index 24950a63..5ad00547 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from logging import Logger from threading import Lock from typing import Dict, List, Optional, Text diff --git a/reportportal_client/core/rp_file.py b/reportportal_client/core/rp_file.py index 9ed10b3d..68ca0f22 100644 --- a/reportportal_client/core/rp_file.py +++ b/reportportal_client/core/rp_file.py @@ -1,5 +1,18 @@ """This module contains classes representing RP file object.""" +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + import uuid diff --git a/reportportal_client/core/rp_file.pyi b/reportportal_client/core/rp_file.pyi index cf681a7a..1d08b606 100644 --- a/reportportal_client/core/rp_file.pyi +++ b/reportportal_client/core/rp_file.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from typing import Any, Dict, Optional, Text class RPFile: diff --git a/reportportal_client/core/rp_issues.py b/reportportal_client/core/rp_issues.py index b013adca..c67cd096 100644 --- a/reportportal_client/core/rp_issues.py +++ b/reportportal_client/core/rp_issues.py @@ -24,22 +24,20 @@ } ... } - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. """ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License class Issue(object): """This class represents an issue that can be attached to test result.""" diff --git a/reportportal_client/core/rp_issues.pyi b/reportportal_client/core/rp_issues.pyi index 7ee80af9..de58bae9 100644 --- a/reportportal_client/core/rp_issues.pyi +++ b/reportportal_client/core/rp_issues.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from typing import Dict, List, Optional, Text class Issue: diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index f49df3af..ee6b813e 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -3,22 +3,21 @@ Detailed information about requests wrapped up in that module can be found by the following link: https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. """ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + import json import logging diff --git a/reportportal_client/core/rp_requests.pyi b/reportportal_client/core/rp_requests.pyi index 879e6eee..75fa5102 100644 --- a/reportportal_client/core/rp_requests.pyi +++ b/reportportal_client/core/rp_requests.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from reportportal_client.core.rp_file import RPFile as RPFile from reportportal_client.core.rp_issues import Issue as Issue from reportportal_client.core.rp_responses import RPResponse as RPResponse diff --git a/reportportal_client/core/rp_responses.py b/reportportal_client/core/rp_responses.py index 34eb6402..a9bfe039 100644 --- a/reportportal_client/core/rp_responses.py +++ b/reportportal_client/core/rp_responses.py @@ -3,21 +3,21 @@ Detailed information about responses wrapped up in that module can be found by the following link: https://github.com/reportportal/documentation/blob/master/src/md/src/DevGuides/reporting.md +""" -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" import logging from reportportal_client.static.defines import NOT_FOUND diff --git a/reportportal_client/core/rp_responses.pyi b/reportportal_client/core/rp_responses.pyi index 234195b9..92d95c05 100644 --- a/reportportal_client/core/rp_responses.pyi +++ b/reportportal_client/core/rp_responses.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + from requests import Response from typing import Dict, Iterable, Optional, Text, Tuple diff --git a/reportportal_client/core/test_manager.py b/reportportal_client/core/test_manager.py index 5e38f0dc..9fdc2ddc 100644 --- a/reportportal_client/core/test_manager.py +++ b/reportportal_client/core/test_manager.py @@ -1,19 +1,17 @@ -"""This module contains functional for test items management. - -Copyright (c) 2018 http://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This module contains functional for test items management.""" + +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License from reportportal_client.helpers import generate_uuid, dict_to_payload from reportportal_client.items.rp_log_items.rp_log_item import RPLogItem diff --git a/reportportal_client/core/worker.py b/reportportal_client/core/worker.py index eb79832c..bca25775 100644 --- a/reportportal_client/core/worker.py +++ b/reportportal_client/core/worker.py @@ -1,19 +1,17 @@ -"""This module contains worker that makes non-blocking HTTP requests. - -Copyright (c) 2022 https://reportportal.io . - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -https://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" +"""This module contains worker that makes non-blocking HTTP requests.""" + +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License import logging import threading diff --git a/reportportal_client/core/worker.pyi b/reportportal_client/core/worker.pyi index d81134a6..7ddb06ac 100644 --- a/reportportal_client/core/worker.pyi +++ b/reportportal_client/core/worker.pyi @@ -1,3 +1,16 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + import threading from logging import Logger from queue import PriorityQueue From d4caaf1a9b8cb4f3b8d34a2e4d83633e3cc68ce8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 18:14:34 +0300 Subject: [PATCH 15/33] Log batch size calculation: WIP --- reportportal_client/core/log_manager.py | 59 ++++++++++++++---------- reportportal_client/core/log_manager.pyi | 10 ++-- reportportal_client/core/rp_requests.py | 8 ++++ reportportal_client/core/rp_requests.pyi | 52 ++++++++++++++++++--- reportportal_client/helpers.py | 46 ++++++++++++++++++ reportportal_client/helpers.pyi | 5 ++ tests/steps/test_steps.py | 20 ++++---- 7 files changed, 154 insertions(+), 46 deletions(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 13bd6276..66b08406 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -18,6 +18,7 @@ from six.moves import queue +import helpers from reportportal_client.core.rp_requests import ( HttpRequest, RPFile, @@ -36,31 +37,32 @@ class LogManager(object): """Manager of the log items.""" def __init__(self, rp_url, session, api_version, launch_id, project_name, - log_batch_size=20, verify_ssl=True, - log_batch_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE): + max_entry_number=20, verify_ssl=True, + max_payload_size=MAX_LOG_BATCH_PAYLOAD_SIZE): """Initialize instance attributes. - :param rp_url: Report portal URL - :param session: HTTP Session object - :param api_version: RP API version - :param launch_id: Parent launch UUID - :param project_name: RP project name - :param log_batch_size: The amount of log objects that need to - be gathered before processing - :param verify_ssl: Indicates that it is necessary to - verify SSL - certificates within HTTP request - :param log_batch_payload_size: maximum size in bytes of logs that can - be processed in one batch + :param rp_url: Report portal URL + :param session: HTTP Session object + :param api_version: RP API version + :param launch_id: Parent launch UUID + :param project_name: RP project name + :param max_entry_number: The amount of log objects that need to be + gathered before processing + :param verify_ssl: Indicates that it is necessary to verify SSL + certificates within HTTP request + :param max_payload_size: maximum size in bytes of logs that can be + processed in one batch """ - self._lock = Lock() - self._logs_batch = [] + self._stop_lock = Lock() + self._size_lock = Lock() + self._batch = [] + self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH self._worker = None self.api_version = api_version self.queue = queue.PriorityQueue() self.launch_id = launch_id - self.log_batch_size = log_batch_size - self.log_batch_payload_size = log_batch_payload_size + self.max_entry_number = max_entry_number + self.max_payload_size = max_payload_size self.project_name = project_name self.rp_url = rp_url self.session = session @@ -76,19 +78,26 @@ def _log_process(self, log_req): :param log_req: RPRequestLog object """ - self._logs_batch.append(log_req) - if len(self._logs_batch) >= self.log_batch_size: - self._send_batch() + with self._size_lock: + rq_size = log_req.multipart_size + if self._payload_size + rq_size >= self.max_payload_size: + if len(self._batch) > 0: + self._send_batch() + + self._batch.append(log_req) + if len(self._batch) >= self.max_entry_number: + self._send_batch() def _send_batch(self): """Send existing batch logs to the worker.""" - batch = RPLogBatch(self._logs_batch) + batch = RPLogBatch(self._batch) http_request = HttpRequest( self.session.post, self._log_endpoint, files=batch.payload, verify_ssl=self.verify_ssl) batch.http_request = http_request self._worker.send(batch) - self._logs_batch.clear() + self._batch.clear() + self.max_payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH def log(self, time, message=None, level=None, attachment=None, item_id=None): @@ -118,8 +127,8 @@ def start(self): def stop(self): """Send last batches to the worker followed by the stop command.""" if self._worker: - with self._lock: - if self._logs_batch: + with self._stop_lock: + if self._batch: self._send_batch() logger.debug('Waiting for worker {0} to complete' 'processing batches.'.format(self._worker)) diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index 5ad00547..ad7cac2c 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -29,19 +29,21 @@ MAX_LOG_BATCH_PAYLOAD_SIZE: int class LogManager: - _lock: Lock = ... + _stop_lock: Lock = ... + _size_lock: Lock = ... _log_endpoint: Text = ... - _logs_batch: List = ... + _batch: List = ... + _payload_size: int = ... _worker: Optional[APIWorker] = ... api_version: Text = ... queue: queue.PriorityQueue = ... launch_id: Text = ... - log_batch_size: int = ... + max_entry_number: int = ... project_name: Text = ... rp_url: Text = ... session: Session = ... verify_ssl: bool = ... - log_batch_payload_size: int = ... + max_payload_size: int = ... def __init__(self, rp_url: Text, diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index ee6b813e..f18b5534 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -21,6 +21,7 @@ import json import logging +import helpers from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue from reportportal_client.helpers import dict_to_payload @@ -419,6 +420,13 @@ def payload(self): payload.update(self.__file()) return payload + @property + def multipart_size(self): + """Calculate request size how it would transfer in Multipart HTTP""" + size = helpers.calculate_json_part_size(self.payload) + size += helpers.calculate_file_part_size(self.file) + return size + class RPLogBatch(RPRequestBase): """RP log save batches with attachments request model. diff --git a/reportportal_client/core/rp_requests.pyi b/reportportal_client/core/rp_requests.pyi index 75fa5102..332cccf3 100644 --- a/reportportal_client/core/rp_requests.pyi +++ b/reportportal_client/core/rp_requests.pyi @@ -11,13 +11,14 @@ # See the License for the specific language governing permissions and # limitations under the License +from typing import Any, Callable, ByteString, Dict, IO, List, Optional, Text, \ + Union, Tuple + from reportportal_client.core.rp_file import RPFile as RPFile from reportportal_client.core.rp_issues import Issue as Issue from reportportal_client.core.rp_responses import RPResponse as RPResponse from reportportal_client.static.abstract import AbstractBaseClass from reportportal_client.static.defines import Priority as Priority -from typing import Any, Callable, ByteString, Dict, IO, List, Optional, Text, \ - Union, Tuple class HttpRequest: @@ -29,14 +30,17 @@ class HttpRequest: verify_ssl: Optional[bool] = ... http_timeout: Union[float, Tuple[float, float]] = ... name: Optional[Text] = ... + def __init__(self, session_method: Callable, url: Text, - data = Optional[Union[Dict, List[Union[tuple, ByteString, IO]]]], - json = Optional[Dict], - files = Optional[Dict], - verify_ssl = Optional[bool], - name = Optional[Text]) -> None: ... + data=Optional[ + Union[Dict, List[Union[tuple, ByteString, IO]]]], + json=Optional[Dict], + files=Optional[Dict], + verify_ssl=Optional[bool], + name=Optional[Text]) -> None: ... + def make(self) -> Optional[RPResponse]: ... @@ -45,22 +49,32 @@ class RPRequestBase(metaclass=AbstractBaseClass): _http_request: Optional[HttpRequest] = ... _priority: Priority = ... _response: Optional[RPResponse] = ... + def __init__(self) -> None: ... + def __lt__(self, other: RPRequestBase) -> bool: ... + @property def http_request(self) -> HttpRequest: ... + @http_request.setter def http_request(self, value: HttpRequest) -> None: ... + @property def priority(self) -> Priority: ... + @priority.setter def priority(self, value: Priority) -> None: ... + @property def response(self) -> Optional[RPResponse]: ... + @response.setter def response(self, value: RPResponse) -> None: ... + def payload(self) -> Dict: ... + class LaunchStartRequest(RPRequestBase): attributes: Optional[Union[List, Dict]] = ... description: Text = ... @@ -70,6 +84,7 @@ class LaunchStartRequest(RPRequestBase): rerun_of: Text = ... start_time: Text = ... uuid: Text = ... + def __init__(self, name: Text, start_time: Text, @@ -79,22 +94,27 @@ class LaunchStartRequest(RPRequestBase): rerun: bool = ..., rerun_of: Optional[Text] = ..., uuid: Optional[Text] = ...) -> None: ... + @property def payload(self) -> Dict: ... + class LaunchFinishRequest(RPRequestBase): attributes: Optional[Union[List, Dict]] = ... description: Text = ... end_time: Text = ... status: Text = ... + def __init__(self, end_time: Text, status: Optional[Text] = ..., attributes: Optional[Union[List, Dict]] = ..., description: Optional[Text] = ...) -> None: ... + @property def payload(self) -> Dict: ... + class ItemStartRequest(RPRequestBase): attributes: Optional[Union[List, Dict]] = ... code_ref: Text = ... @@ -109,6 +129,7 @@ class ItemStartRequest(RPRequestBase): type_: Text = ... uuid: Text = ... unique_id: Text = ... + def __init__(self, name: Text, start_time: Text, @@ -123,9 +144,11 @@ class ItemStartRequest(RPRequestBase): test_case_id: Optional[Text] = ..., uuid: Optional[Any] = ..., unique_id: Optional[Any] = ...) -> None: ... + @property def payload(self) -> Dict: ... + class ItemFinishRequest(RPRequestBase): attributes: Optional[Union[List, Dict]] = ... description: Text = ... @@ -135,6 +158,7 @@ class ItemFinishRequest(RPRequestBase): launch_uuid: Text = ... status: Text = ... retry: bool = ... + def __init__(self, end_time: Text, launch_uuid: Text, @@ -144,9 +168,11 @@ class ItemFinishRequest(RPRequestBase): is_skipped_an_issue: bool = ..., issue: Optional[Issue] = ..., retry: bool = ...) -> None: ... + @property def payload(self) -> Dict: ... + class RPRequestLog(RPRequestBase): file: RPFile = ... launch_uuid: Text = ... @@ -154,6 +180,7 @@ class RPRequestLog(RPRequestBase): message: Text = ... time: Text = ... item_uuid: Text = ... + def __init__(self, launch_uuid: Text, time: Text, @@ -161,16 +188,27 @@ class RPRequestLog(RPRequestBase): item_uuid: Optional[Text] = ..., level: Text = ..., message: Optional[Text] = ...) -> None: ... + def __file(self) -> Dict: ... + @property def payload(self) -> Dict: ... + @property + def multipart_size(self) -> int: ... + + class RPLogBatch(RPRequestBase): default_content: Text = ... log_reqs: List[RPRequestLog] = ... + def __init__(self, log_reqs: List[RPRequestLog]) -> None: ... + def __get_file(self, rp_file: RPFile) -> tuple: ... + def __get_files(self) -> List: ... + def __get_request_part(self) -> Dict: ... + @property def payload(self) -> Dict: ... diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index 14c6e2fa..e385b006 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -12,6 +12,7 @@ limitations under the License. """ import inspect +import json import logging import time import uuid @@ -182,3 +183,48 @@ def get_function_params(func, args, kwargs): for arg_name, arg_value in kwargs.items(): result[arg_name] = arg_value return result if len(result.items()) > 0 else None + + +TYPICAL_MULTIPART_BOUNDARY = '--972dbca3abacfd01fb4aea0571532b52' + +TYPICAL_JSON_PART_HEADER = TYPICAL_MULTIPART_BOUNDARY + '''\r +Content-Disposition: form-data; name="json_request_part"\r +Content-Type: application/json\r +\r +''' + +TYPICAL_FILE_PART_HEADER = TYPICAL_MULTIPART_BOUNDARY + '''\r +Content-Disposition: form-data; name="file"; filename="{0}"\r +Content-Type: {1}\r +\r +''' + +TYPICAL_JSON_PART_HEADER_LENGTH = len(TYPICAL_JSON_PART_HEADER) + +TYPICAL_MULTIPART_FOOTER = '\r\n' + TYPICAL_MULTIPART_BOUNDARY + '--' + +TYPICAL_MULTIPART_FOOTER_LENGTH = len(TYPICAL_MULTIPART_FOOTER) + +TYPICAL_JSON_ARRAY = "[]" + +TYPICAL_JSON_ARRAY_LENGTH = len(TYPICAL_JSON_ARRAY) + +TYPICAL_JSON_ARRAY_ELEMENT = "," + +TYPICAL_JSON_ARRAY_ELEMENT_LENGTH = len(TYPICAL_JSON_ARRAY_ELEMENT) + + +def calculate_json_part_size(json_dict): + size = len(json.dumps(json_dict)) + size += TYPICAL_JSON_PART_HEADER_LENGTH + size += TYPICAL_JSON_ARRAY_LENGTH + size += TYPICAL_JSON_ARRAY_ELEMENT_LENGTH + return size + + +def calculate_file_part_size(file): + if file is None: + return 0 + size = len(TYPICAL_FILE_PART_HEADER.format(file.name, file.content_type)) + size += len(file.content) + return size diff --git a/reportportal_client/helpers.pyi b/reportportal_client/helpers.pyi index b084543b..cbfe50fb 100644 --- a/reportportal_client/helpers.pyi +++ b/reportportal_client/helpers.pyi @@ -1,3 +1,4 @@ +from core.rp_file import RPFile from .errors import EntryCreatedError as EntryCreatedError, \ OperationCompletionError as OperationCompletionError, \ ResponseError as ResponseError @@ -8,6 +9,8 @@ from requests import Response logger: Logger +TYPICAL_MULTIPART_FOOTER_LENGTH: int + def generate_uuid() -> Text: ... def convert_string(value: Text) -> Text: ... def dict_to_payload(dictionary: dict) -> list[dict]: ... @@ -24,3 +27,5 @@ def verify_value_length(attributes: list[dict]) -> list[dict]: ... def timestamp() -> Text: ... def get_function_params(func: Callable, args: Tuple[Any, ...], kwargs: Dict[Text, Any]) -> Dict[Text, Any]: ... +def calculate_json_part_size(json_dict: dict) -> int: ... +def calculate_file_part_size(file: RPFile) -> int: ... \ No newline at end of file diff --git a/tests/steps/test_steps.py b/tests/steps/test_steps.py index 98a853a0..d31fcb87 100644 --- a/tests/steps/test_steps.py +++ b/tests/steps/test_steps.py @@ -81,7 +81,7 @@ def test_nested_step_decorator(rp_client): assert rp_client.session.post.call_count == 1 assert rp_client.session.put.call_count == 1 - assert len(rp_client._log_manager._logs_batch) == 0 + assert len(rp_client._log_manager._batch) == 0 def test_nested_step_failed(rp_client): @@ -125,24 +125,24 @@ def nested_step_params(param1, param2, param3=None): def test_verify_parameters_logging_default_value(rp_client): rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two') - assert len(rp_client._log_manager._logs_batch) == 1 - assert rp_client._log_manager._logs_batch[0].message \ + assert len(rp_client._log_manager._batch) == 1 + assert rp_client._log_manager._batch[0].message \ == "Parameters: param1: 1; param2: two" def test_verify_parameters_logging_no_default_value(rp_client): rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two', 'three') - assert len(rp_client._log_manager._logs_batch) == 1 - assert rp_client._log_manager._logs_batch[0].message \ + assert len(rp_client._log_manager._batch) == 1 + assert rp_client._log_manager._batch[0].message \ == "Parameters: param1: 1; param2: two; param3: three" def test_verify_parameters_logging_named_value(rp_client): rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two', param3='three') - assert len(rp_client._log_manager._logs_batch) == 1 - assert rp_client._log_manager._logs_batch[0].message \ + assert len(rp_client._log_manager._batch) == 1 + assert rp_client._log_manager._batch[0].message \ == "Parameters: param1: 1; param2: two; param3: three" @@ -150,8 +150,8 @@ def test_verify_parameters_inline_logging(rp_client): rp_client._item_stack.append(PARENT_STEP_ID) with step(NESTED_STEP_NAME, params={'param1': 1, 'param2': 'two'}): pass - assert len(rp_client._log_manager._logs_batch) == 1 - assert rp_client._log_manager._logs_batch[0].message \ + assert len(rp_client._log_manager._batch) == 1 + assert rp_client._log_manager._batch[0].message \ == "Parameters: param1: 1; param2: two" @@ -177,7 +177,7 @@ def test_two_level_nested_step_decorator(rp_client): assert rp_client.session.post.call_count == 2 assert rp_client.session.put.call_count == 2 - assert len(rp_client._log_manager._logs_batch) == 0 + assert len(rp_client._log_manager._batch) == 0 request_uri = rp_client.session.post.call_args_list[0][0][0] first_parent_id = request_uri[request_uri.rindex('/') + 1:] From b773acbc52d1cddcbd6ef26cd8b2ba8ffd204e3b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 19:57:37 +0300 Subject: [PATCH 16/33] Build fix --- reportportal_client/core/rp_requests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index f18b5534..365a0034 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -21,7 +21,7 @@ import json import logging -import helpers +from reportportal_client import helpers from reportportal_client.core.rp_file import RPFile from reportportal_client.core.rp_issues import Issue from reportportal_client.helpers import dict_to_payload From 072595af679af5f80aa1805b58390a2f6aa24369 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 20:21:42 +0300 Subject: [PATCH 17/33] Build fix --- reportportal_client/core/log_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 66b08406..283e1092 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -18,7 +18,7 @@ from six.moves import queue -import helpers +from reportportal_client import helpers from reportportal_client.core.rp_requests import ( HttpRequest, RPFile, From 1b11ac391645baf9161640ee045a42849bd71aa9 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 20:26:17 +0300 Subject: [PATCH 18/33] Build fix --- reportportal_client/client.py | 2 +- reportportal_client/core/log_manager.pyi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 48a71d18..29ad5683 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -115,7 +115,7 @@ def __init__(self, self._log_manager = LogManager( self.endpoint, self.session, self.api_v2, self.launch_id, self.project, log_batch_size=log_batch_size, - log_batch_payload_size=log_batch_payload_size) + max_payload_size=log_batch_payload_size) def finish_launch(self, end_time, diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index ad7cac2c..94276362 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -53,7 +53,7 @@ class LogManager: project_name: Text, log_batch_size: int = ..., verify_ssl: bool = ..., - log_batch_payload_size: int = ...) -> None: ... + max_payload_size: int = ...) -> None: ... def _log_process(self, log_req: RPRequestLog) -> None: ... From d96356dd43d898618ac4d0438a78b1ef5bdbb0d8 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 20:30:54 +0300 Subject: [PATCH 19/33] Build fix --- reportportal_client/client.py | 2 +- reportportal_client/core/log_manager.pyi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/client.py b/reportportal_client/client.py index 29ad5683..f7e5a226 100644 --- a/reportportal_client/client.py +++ b/reportportal_client/client.py @@ -114,7 +114,7 @@ def __init__(self, self._log_manager = LogManager( self.endpoint, self.session, self.api_v2, self.launch_id, - self.project, log_batch_size=log_batch_size, + self.project, max_entry_number=log_batch_size, max_payload_size=log_batch_payload_size) def finish_launch(self, diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index 94276362..ab3aa2ba 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -51,7 +51,7 @@ class LogManager: api_version: Text, launch_id: Text, project_name: Text, - log_batch_size: int = ..., + max_entry_number: int = ..., verify_ssl: bool = ..., max_payload_size: int = ...) -> None: ... From dd94a8132913f93d921b06fd0b83f150b59584f0 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 20:50:36 +0300 Subject: [PATCH 20/33] Build fix --- tests/steps/test_steps.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/tests/steps/test_steps.py b/tests/steps/test_steps.py index d31fcb87..32fb2cf4 100644 --- a/tests/steps/test_steps.py +++ b/tests/steps/test_steps.py @@ -117,12 +117,23 @@ def test_nested_step_custom_status_failed(rp_client): assert rp_client.session.put.call_args[1]['json']['status'] == 'FAILED' +def item_id_gen(*args, **kwargs): + item_id = 'post-{}-{}'.format( + str(round(time.time() * 1000)), + random.randint(0, 9999)) + result = mock.Mock() + result.text = '{{"id": "{}"}}'.format(item_id) + result.json = lambda: {'id': item_id} + return result + + @step def nested_step_params(param1, param2, param3=None): pass def test_verify_parameters_logging_default_value(rp_client): + rp_client.session.post.side_effect = item_id_gen rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two') assert len(rp_client._log_manager._batch) == 1 @@ -131,6 +142,7 @@ def test_verify_parameters_logging_default_value(rp_client): def test_verify_parameters_logging_no_default_value(rp_client): + rp_client.session.post.side_effect = item_id_gen rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two', 'three') assert len(rp_client._log_manager._batch) == 1 @@ -139,6 +151,7 @@ def test_verify_parameters_logging_no_default_value(rp_client): def test_verify_parameters_logging_named_value(rp_client): + rp_client.session.post.side_effect = item_id_gen rp_client._item_stack.append(PARENT_STEP_ID) nested_step_params(1, 'two', param3='three') assert len(rp_client._log_manager._batch) == 1 @@ -147,6 +160,7 @@ def test_verify_parameters_logging_named_value(rp_client): def test_verify_parameters_inline_logging(rp_client): + rp_client.session.post.side_effect = item_id_gen rp_client._item_stack.append(PARENT_STEP_ID) with step(NESTED_STEP_NAME, params={'param1': 1, 'param2': 'two'}): pass @@ -155,16 +169,6 @@ def test_verify_parameters_inline_logging(rp_client): == "Parameters: param1: 1; param2: two" -def item_id_gen(*args, **kwargs): - item_id = 'post-{}-{}'.format( - str(round(time.time() * 1000)), - random.randint(0, 9999)) - result = mock.Mock() - result.text = '{{"id": "{}"}}'.format(item_id) - result.json = lambda: {'id': item_id} - return result - - @step def parent_nested_step(): nested_step() From 1d1095f3ee93f8b031eb1128f7bd8bdf1098655f Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 20:53:14 +0300 Subject: [PATCH 21/33] Build fix --- reportportal_client/core/rp_issues.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reportportal_client/core/rp_issues.py b/reportportal_client/core/rp_issues.py index c67cd096..1449a2bc 100644 --- a/reportportal_client/core/rp_issues.py +++ b/reportportal_client/core/rp_issues.py @@ -39,6 +39,7 @@ # See the License for the specific language governing permissions and # limitations under the License + class Issue(object): """This class represents an issue that can be attached to test result.""" From 7cdcd2868916afc5c7aa72e85e5d1b8a14dbf272 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 21:00:09 +0300 Subject: [PATCH 22/33] Pydocs --- reportportal_client/core/rp_requests.py | 2 +- reportportal_client/helpers.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/reportportal_client/core/rp_requests.py b/reportportal_client/core/rp_requests.py index 365a0034..8585b252 100644 --- a/reportportal_client/core/rp_requests.py +++ b/reportportal_client/core/rp_requests.py @@ -422,7 +422,7 @@ def payload(self): @property def multipart_size(self): - """Calculate request size how it would transfer in Multipart HTTP""" + """Calculate request size how it would transfer in Multipart HTTP.""" size = helpers.calculate_json_part_size(self.payload) size += helpers.calculate_file_part_size(self.file) return size diff --git a/reportportal_client/helpers.py b/reportportal_client/helpers.py index e385b006..e66e9829 100644 --- a/reportportal_client/helpers.py +++ b/reportportal_client/helpers.py @@ -215,6 +215,11 @@ def get_function_params(func, args, kwargs): def calculate_json_part_size(json_dict): + """Predict a JSON part size of Multipart request. + + :param json_dict: a dictionary representing the JSON + :return: Multipart request part size + """ size = len(json.dumps(json_dict)) size += TYPICAL_JSON_PART_HEADER_LENGTH size += TYPICAL_JSON_ARRAY_LENGTH @@ -223,6 +228,11 @@ def calculate_json_part_size(json_dict): def calculate_file_part_size(file): + """Predict a file part size of Multipart request. + + :param file: RPFile class instance + :return: Multipart request part size + """ if file is None: return 0 size = len(TYPICAL_FILE_PART_HEADER.format(file.name, file.content_type)) From 2124bd9da812ff6eb4031e41949b5d70ac1a61b7 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Wed, 22 Jun 2022 21:21:42 +0300 Subject: [PATCH 23/33] EOL fix --- reportportal_client/helpers.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/helpers.pyi b/reportportal_client/helpers.pyi index cbfe50fb..527d08a7 100644 --- a/reportportal_client/helpers.pyi +++ b/reportportal_client/helpers.pyi @@ -28,4 +28,4 @@ def timestamp() -> Text: ... def get_function_params(func: Callable, args: Tuple[Any, ...], kwargs: Dict[Text, Any]) -> Dict[Text, Any]: ... def calculate_json_part_size(json_dict: dict) -> int: ... -def calculate_file_part_size(file: RPFile) -> int: ... \ No newline at end of file +def calculate_file_part_size(file: RPFile) -> int: ... From b853b3efc1dac519418aea9b3e7c99e28ec3523c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 09:49:36 +0300 Subject: [PATCH 24/33] Test correction --- tests/logs/test_rp_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/logs/test_rp_file.py b/tests/logs/test_rp_file.py index 7c526a62..4af5e60f 100644 --- a/tests/logs/test_rp_file.py +++ b/tests/logs/test_rp_file.py @@ -25,7 +25,7 @@ ] ) def test_rp_file_name_should_not_be_empty(name): - file = RPFile(name, '{"test": true}', 'application/json') + file = RPFile(name, b'{"test": true}', 'application/json') payload = file.payload assert payload['name'] From 08dae1eec72e59602df179f5cbce8fcc77572720 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 09:50:14 +0300 Subject: [PATCH 25/33] Fix payload size tracking --- reportportal_client/core/log_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 283e1092..d849b95d 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -83,8 +83,8 @@ def _log_process(self, log_req): if self._payload_size + rq_size >= self.max_payload_size: if len(self._batch) > 0: self._send_batch() - self._batch.append(log_req) + self.max_payload_size += rq_size if len(self._batch) >= self.max_entry_number: self._send_batch() From 4bc3d7edfbee56b7b93c8225d764e2ebc7fac67a Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 09:51:45 +0300 Subject: [PATCH 26/33] Move '_send_batch' method upper --- reportportal_client/core/log_manager.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index d849b95d..e7cc8218 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -73,6 +73,17 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, .format(rp_url=rp_url, version=self.api_version, project_name=self.project_name)) + def _send_batch(self): + """Send existing batch logs to the worker.""" + batch = RPLogBatch(self._batch) + http_request = HttpRequest( + self.session.post, self._log_endpoint, files=batch.payload, + verify_ssl=self.verify_ssl) + batch.http_request = http_request + self._worker.send(batch) + self._batch.clear() + self.max_payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + def _log_process(self, log_req): """Process the given log request. @@ -88,17 +99,6 @@ def _log_process(self, log_req): if len(self._batch) >= self.max_entry_number: self._send_batch() - def _send_batch(self): - """Send existing batch logs to the worker.""" - batch = RPLogBatch(self._batch) - http_request = HttpRequest( - self.session.post, self._log_endpoint, files=batch.payload, - verify_ssl=self.verify_ssl) - batch.http_request = http_request - self._worker.send(batch) - self._batch.clear() - self.max_payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH - def log(self, time, message=None, level=None, attachment=None, item_id=None): """Log message. Can be added to test item in any state. From a4d7b3ca82bedfc1e7eeb2bbc3465a01f5b5c8b5 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 10:05:35 +0300 Subject: [PATCH 27/33] CHANGELOG.md update --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44a2d7d1..75dbd296 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## [Unreleased] ### Added - Ability to pass client instance in `RPLogHandler` constructor, by @HardNorth +- Issue [#179](https://github.com/reportportal/client-Python/issues/179): batch logging request payload size tracking, by @HardNorth +### Fixed +- Issue [#184](https://github.com/reportportal/client-Python/issues/184): early logger initialization exception, by @dagansandler ## [5.2.2] ### Fixed From c83650ba3c086b8d3faac87d0b154c9d9fc1a28b Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 12:03:35 +0300 Subject: [PATCH 28/33] Fixes after testing --- reportportal_client/core/log_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index e7cc8218..49059237 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -75,14 +75,15 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, def _send_batch(self): """Send existing batch logs to the worker.""" - batch = RPLogBatch(self._batch) + # copy list to not modify batch by `self._batch.clear()` + batch = RPLogBatch(list(self._batch)) http_request = HttpRequest( self.session.post, self._log_endpoint, files=batch.payload, verify_ssl=self.verify_ssl) batch.http_request = http_request self._worker.send(batch) self._batch.clear() - self.max_payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH def _log_process(self, log_req): """Process the given log request. @@ -95,7 +96,7 @@ def _log_process(self, log_req): if len(self._batch) > 0: self._send_batch() self._batch.append(log_req) - self.max_payload_size += rq_size + self._payload_size += rq_size if len(self._batch) >= self.max_entry_number: self._send_batch() From 0d19d4ebde40d595833ef6afd7892fb78fc08843 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 12:04:03 +0300 Subject: [PATCH 29/33] LogManager tests --- tests/logs/test_log_manager.py | 91 ++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 tests/logs/test_log_manager.py diff --git a/tests/logs/test_log_manager.py b/tests/logs/test_log_manager.py new file mode 100644 index 00000000..e14d2e3b --- /dev/null +++ b/tests/logs/test_log_manager.py @@ -0,0 +1,91 @@ +# Copyright (c) 2022 EPAM Systems +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + + +from six.moves import mock + +from reportportal_client import helpers +from reportportal_client.core.log_manager import LogManager + +RP_URL = 'http://docker.local:8080' +API_VERSION = 'api/v2' +TEST_LAUNCH_ID = 'test_launch_id' +TEST_ITEM_ID = 'test_item_id' +PROJECT_NAME = 'test_project' +KILOBYTE = 2 ** 10 +MEGABYTE = KILOBYTE ** KILOBYTE +ABOVE_LIMIT_SIZE = MEGABYTE * 65 +TEST_MASSAGE = 'test_message' +TEST_LEVEL = 'DEBUG' +TEST_BATCH_SIZE = 5 + + +# noinspection PyUnresolvedReferences +def test_log_batch_send_by_length(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + for _ in range(TEST_BATCH_SIZE): + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + + assert log_manager._worker.send.call_count == 1 + batch = log_manager._worker.send.call_args[0][0] + assert len(batch.log_reqs) == 5 + assert batch.http_request is not None + assert 'post' in session._mock_children + assert len(log_manager._batch) == 0 + assert log_manager._payload_size == helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + + +# noinspection PyUnresolvedReferences +def test_log_batch_not_send_by_length(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + for _ in range(TEST_BATCH_SIZE - 1): + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + + assert log_manager._worker.send.call_count == 0 + assert 'post' not in session._mock_children + assert len(log_manager._batch) == 4 + assert log_manager._payload_size > helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + + +# noinspection PyUnresolvedReferences +def test_log_batch_send_by_stop(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + for _ in range(TEST_BATCH_SIZE - 1): + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + log_manager.stop() + + assert log_manager._worker.send.call_count == 1 + batch = log_manager._worker.send.call_args[0][0] + assert len(batch.log_reqs) == 4 + assert batch.http_request is not None + assert 'post' in session._mock_children + assert len(log_manager._batch) == 0 + assert log_manager._payload_size == helpers.TYPICAL_MULTIPART_FOOTER_LENGTH From 074bb04ab2c5b3e8e1120a3c342cb71343521270 Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 13:01:18 +0300 Subject: [PATCH 30/33] LogManager tests --- reportportal_client/helpers.pyi | 1 + tests/logs/test_log_manager.py | 93 +++++++++++++++++++++++++++++++-- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/reportportal_client/helpers.pyi b/reportportal_client/helpers.pyi index 527d08a7..f60364ce 100644 --- a/reportportal_client/helpers.pyi +++ b/reportportal_client/helpers.pyi @@ -9,6 +9,7 @@ from requests import Response logger: Logger +TYPICAL_FILE_PART_HEADER: Text TYPICAL_MULTIPART_FOOTER_LENGTH: int def generate_uuid() -> Text: ... diff --git a/tests/logs/test_log_manager.py b/tests/logs/test_log_manager.py index e14d2e3b..0a5b6c3b 100644 --- a/tests/logs/test_log_manager.py +++ b/tests/logs/test_log_manager.py @@ -11,23 +11,24 @@ # See the License for the specific language governing permissions and # limitations under the License +import os from six.moves import mock from reportportal_client import helpers -from reportportal_client.core.log_manager import LogManager +from reportportal_client.core.log_manager import LogManager, \ + MAX_LOG_BATCH_PAYLOAD_SIZE RP_URL = 'http://docker.local:8080' API_VERSION = 'api/v2' TEST_LAUNCH_ID = 'test_launch_id' TEST_ITEM_ID = 'test_item_id' PROJECT_NAME = 'test_project' -KILOBYTE = 2 ** 10 -MEGABYTE = KILOBYTE ** KILOBYTE -ABOVE_LIMIT_SIZE = MEGABYTE * 65 TEST_MASSAGE = 'test_message' TEST_LEVEL = 'DEBUG' TEST_BATCH_SIZE = 5 +TEST_ATTACHMENT_NAME = 'test_file.bin' +TEST_ATTACHMENT_TYPE = 'application/zip' # noinspection PyUnresolvedReferences @@ -89,3 +90,87 @@ def test_log_batch_send_by_stop(): assert 'post' in session._mock_children assert len(log_manager._batch) == 0 assert log_manager._payload_size == helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + + +# noinspection PyUnresolvedReferences +def test_log_batch_not_send_by_size(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + headers_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH - len( + helpers.TYPICAL_FILE_PART_HEADER.format(TEST_ATTACHMENT_NAME, + TEST_ATTACHMENT_TYPE)) + attachment_size = MAX_LOG_BATCH_PAYLOAD_SIZE - headers_size - 1024 + random_byte_array = bytearray(os.urandom(attachment_size)) + attachment = {'name': TEST_ATTACHMENT_NAME, 'content': random_byte_array, + 'content_type': TEST_ATTACHMENT_TYPE} + + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID, attachment=attachment) + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + + assert log_manager._worker.send.call_count == 0 + assert 'post' not in session._mock_children + assert len(log_manager._batch) == 2 + assert log_manager._payload_size > MAX_LOG_BATCH_PAYLOAD_SIZE - 1024 + assert log_manager._payload_size < MAX_LOG_BATCH_PAYLOAD_SIZE + + +# noinspection PyUnresolvedReferences +def test_log_batch_send_by_size(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + random_byte_array = bytearray(os.urandom(MAX_LOG_BATCH_PAYLOAD_SIZE)) + attachment = {'name': TEST_ATTACHMENT_NAME, 'content': random_byte_array, + 'content_type': TEST_ATTACHMENT_TYPE} + + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID, attachment=attachment) + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + + assert log_manager._worker.send.call_count == 1 + batch = log_manager._worker.send.call_args[0][0] + assert len(batch.log_reqs) == 1 + assert batch.http_request is not None + assert 'post' in session._mock_children + assert len(log_manager._batch) == 1 + assert log_manager._payload_size < \ + helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + 1024 + + +# noinspection PyUnresolvedReferences +def test_log_batch_triggers_previous_request_to_send(): + session = mock.Mock() + log_manager = LogManager(RP_URL, session, API_VERSION, TEST_LAUNCH_ID, + PROJECT_NAME, max_entry_number=TEST_BATCH_SIZE, + verify_ssl=False) + log_manager._worker = mock.Mock() + + random_byte_array = bytearray(os.urandom(MAX_LOG_BATCH_PAYLOAD_SIZE)) + attachment = {'name': TEST_ATTACHMENT_NAME, 'content': random_byte_array, + 'content_type': TEST_ATTACHMENT_TYPE} + + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID) + payload_size = log_manager._payload_size + assert payload_size < helpers.TYPICAL_MULTIPART_FOOTER_LENGTH + 1024 + + log_manager.log(helpers.timestamp(), TEST_MASSAGE, TEST_LEVEL, + item_id=TEST_ITEM_ID, attachment=attachment) + + assert log_manager._worker.send.call_count == 1 + batch = log_manager._worker.send.call_args[0][0] + assert len(batch.log_reqs) == 1 + assert batch.http_request is not None + assert 'post' in session._mock_children + assert len(log_manager._batch) == 1 + assert log_manager._payload_size > MAX_LOG_BATCH_PAYLOAD_SIZE From 875fc67aa1a6b607f8dd80ba3bbe15600c704c9c Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 13:05:39 +0300 Subject: [PATCH 31/33] Python 2.7 fixes --- reportportal_client/core/log_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 49059237..c0d49ba0 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -76,13 +76,13 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, def _send_batch(self): """Send existing batch logs to the worker.""" # copy list to not modify batch by `self._batch.clear()` - batch = RPLogBatch(list(self._batch)) + batch = RPLogBatch(self._batch) http_request = HttpRequest( self.session.post, self._log_endpoint, files=batch.payload, verify_ssl=self.verify_ssl) batch.http_request = http_request self._worker.send(batch) - self._batch.clear() + self._batch = [] self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH def _log_process(self, log_req): From 2ff8214d56c4ae808fdcf563ad050503f4d818ef Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 13:47:28 +0300 Subject: [PATCH 32/33] Remove comment --- reportportal_client/core/log_manager.py | 1 - 1 file changed, 1 deletion(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index c0d49ba0..81acc060 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -75,7 +75,6 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, def _send_batch(self): """Send existing batch logs to the worker.""" - # copy list to not modify batch by `self._batch.clear()` batch = RPLogBatch(self._batch) http_request = HttpRequest( self.session.post, self._log_endpoint, files=batch.payload, From 2d7980b9c19303ebdc0470b21fc2f03d99a4282e Mon Sep 17 00:00:00 2001 From: Vadzim Hushchanskou Date: Thu, 23 Jun 2022 13:49:45 +0300 Subject: [PATCH 33/33] Remove redundant lock --- reportportal_client/core/log_manager.py | 7 +++---- reportportal_client/core/log_manager.pyi | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/reportportal_client/core/log_manager.py b/reportportal_client/core/log_manager.py index 81acc060..2199a158 100644 --- a/reportportal_client/core/log_manager.py +++ b/reportportal_client/core/log_manager.py @@ -53,8 +53,7 @@ def __init__(self, rp_url, session, api_version, launch_id, project_name, :param max_payload_size: maximum size in bytes of logs that can be processed in one batch """ - self._stop_lock = Lock() - self._size_lock = Lock() + self._lock = Lock() self._batch = [] self._payload_size = helpers.TYPICAL_MULTIPART_FOOTER_LENGTH self._worker = None @@ -89,7 +88,7 @@ def _log_process(self, log_req): :param log_req: RPRequestLog object """ - with self._size_lock: + with self._lock: rq_size = log_req.multipart_size if self._payload_size + rq_size >= self.max_payload_size: if len(self._batch) > 0: @@ -127,7 +126,7 @@ def start(self): def stop(self): """Send last batches to the worker followed by the stop command.""" if self._worker: - with self._stop_lock: + with self._lock: if self._batch: self._send_batch() logger.debug('Waiting for worker {0} to complete' diff --git a/reportportal_client/core/log_manager.pyi b/reportportal_client/core/log_manager.pyi index ab3aa2ba..8ab3a516 100644 --- a/reportportal_client/core/log_manager.pyi +++ b/reportportal_client/core/log_manager.pyi @@ -29,8 +29,7 @@ MAX_LOG_BATCH_PAYLOAD_SIZE: int class LogManager: - _stop_lock: Lock = ... - _size_lock: Lock = ... + _lock: Lock = ... _log_endpoint: Text = ... _batch: List = ... _payload_size: int = ...