From db28eecff052daada7ee3f4c6240415fa3e2952b Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 13:25:24 -0700 Subject: [PATCH 1/8] pyupgrade python3.12 --- CHANGELOG.md | 8 +++++++- README.md | 20 ++++++++++---------- example/connexion-example/hello.py | 4 ++-- example/custom_log_format_request.py | 6 +++--- json_logging/__init__.py | 1 - json_logging/dto.py | 6 +++--- json_logging/formatters.py | 15 ++++++--------- json_logging/framework/__init__.py | 1 - json_logging/framework/connexion/__init__.py | 1 - json_logging/framework/fastapi/__init__.py | 1 - json_logging/framework/flask/__init__.py | 1 - json_logging/framework/quart/__init__.py | 1 - json_logging/framework/sanic/__init__.py | 1 - json_logging/framework_base.py | 1 - json_logging/util.py | 3 +-- setup.py | 12 ++---------- tests-performance/benmark_micro.py | 1 - tests/helpers/handler.py | 2 +- 18 files changed, 35 insertions(+), 50 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3b82fa..730bec0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,15 @@ All notable changes to this project will be documented in this file. -This project adheres to [Semantic Versioning](http://semver.org/). +This project adheres to [Semantic Versioning](http://semver.org/). The format is based on [Keep a Changelog](http://keepachangelog.com/). +## [Unreleased] + +### Changed + +- refactor for Python 3.12 + ## 1.4.1rc0 - 2021-04-23 - fix #77 Implement extra properties from json-logging-py diff --git a/README.md b/README.md index 2473d9d..5c0f498 100644 --- a/README.md +++ b/README.md @@ -12,13 +12,13 @@ author. # Content 1. [Features](#1-features) -2. [Usage](#2-usage) - 2.1 [Non-web application log](#21-non-web-application-log) - 2.2 [Web application log](#22-web-application-log) - 2.3 [Get current correlation-id](#23-get-current-correlation-id) - 2.4 [Log extra properties](#24-log-extra-properties) - 2.5 [Root logger](#25-root-logger) - 2.6 [Custom log formatter](#26-custom-log-formatter) +2. [Usage](#2-usage) + 2.1 [Non-web application log](#21-non-web-application-log) + 2.2 [Web application log](#22-web-application-log) + 2.3 [Get current correlation-id](#23-get-current-correlation-id) + 2.4 [Log extra properties](#24-log-extra-properties) + 2.5 [Root logger](#25-root-logger) + 2.6 [Custom log formatter](#26-custom-log-formatter) 2.7 [Exclude certain URL from request instrumentation](#27-exclude-certain-url-from-request-instrumentation) 3. [Configuration](#3-configuration) 4. [Python References](#4-python-references) @@ -259,8 +259,8 @@ json_logging.config_root_logger() ## 2.6 Custom log formatter Customer JSON log formatter can be passed to init method. see examples for more -detail: [non web](https://github.com/thangbn/json-logging-python/blob/master/example/custom_log_format.py), -[web](https://github.com/thangbn/json-logging-python/blob/master/example/custom_log_format_request.py) +detail: [non web](https://github.com/virajkanwade/json-logging-python/blob/master/example/custom_log_format.py), +[web](https://github.com/virajkanwade/json-logging-python/blob/master/example/custom_log_format_request.py) ## 2.7 Exclude certain URl from request instrumentation @@ -298,7 +298,7 @@ TODO: update Python API docs on Github page To add support for a new web framework, you need to extend following classes in [** framework_base**](/blob/master/json_logging/framework_base.py) and register support using [** -json_logging.register_framework_support**](https://github.com/thangbn/json-logging-python/blob/master/json_logging/__init__.py#L38) +json_logging.register_framework_support**](https://github.com/virajkanwade/json-logging-python/blob/master/json_logging/__init__.py#L38) method: Class | Description | Mandatory diff --git a/example/connexion-example/hello.py b/example/connexion-example/hello.py index 78837f5..09b9dec 100755 --- a/example/connexion-example/hello.py +++ b/example/connexion-example/hello.py @@ -14,11 +14,11 @@ def post_greeting(name): logger.info("test log statement") logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) - return 'Hello {name}'.format(name=name) + return f'Hello {name}' def exclude_from_request_instrumentation(name): - return 'Hello {name}. this request wont log request instrumentation information'.format(name=name) + return f'Hello {name}. this request wont log request instrumentation information' def create(): diff --git a/example/custom_log_format_request.py b/example/custom_log_format_request.py index e753267..a766e20 100644 --- a/example/custom_log_format_request.py +++ b/example/custom_log_format_request.py @@ -19,7 +19,7 @@ def _format_log_object(self, record, request_util): request = record.request_response_data._request response = record.request_response_data._response - json_log_object = super(CustomRequestJSONLog, self)._format_log_object(record, request_util) + json_log_object = super()._format_log_object(record, request_util) json_log_object.update({ "customized_prop": "customized value", }) @@ -32,10 +32,10 @@ class CustomDefaultRequestResponseDTO(json_logging.dto.DefaultRequestResponseDTO """ def __init__(self, request, **kwargs): - super(CustomDefaultRequestResponseDTO, self).__init__(request, **kwargs) + super().__init__(request, **kwargs) def on_request_complete(self, response): - super(CustomDefaultRequestResponseDTO, self).on_request_complete(response) + super().on_request_complete(response) self.status = response.status diff --git a/json_logging/__init__.py b/json_logging/__init__.py index 94ee039..c2a4d2d 100644 --- a/json_logging/__init__.py +++ b/json_logging/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 import json import logging import sys diff --git a/json_logging/dto.py b/json_logging/dto.py index 4cf39a8..5d30041 100644 --- a/json_logging/dto.py +++ b/json_logging/dto.py @@ -14,7 +14,7 @@ def __init__(self, request, **kwargs): invoked when request start, where to extract any necessary information from the request object :param request: request object """ - super(RequestResponseDTOBase, self).__init__(**kwargs) + super().__init__(**kwargs) self._request = request def on_request_complete(self, response): @@ -31,14 +31,14 @@ class DefaultRequestResponseDTO(RequestResponseDTOBase): """ def __init__(self, request, **kwargs): - super(DefaultRequestResponseDTO, self).__init__(request, **kwargs) + super().__init__(request, **kwargs) utcnow = datetime.utcnow() self._request_start = utcnow self["request_received_at"] = util.iso_time_format(utcnow) # noinspection PyAttributeOutsideInit def on_request_complete(self, response): - super(DefaultRequestResponseDTO, self).on_request_complete(response) + super().on_request_complete(response) utcnow = datetime.utcnow() time_delta = utcnow - self._request_start self["response_time_ms"] = int(time_delta.total_seconds()) * 1000 + int(time_delta.microseconds / 1000) diff --git a/json_logging/formatters.py b/json_logging/formatters.py index d0a428a..62f784e 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -22,11 +22,8 @@ except NameError: basestring = str -if sys.version_info < (3, 0): - EASY_SERIALIZABLE_TYPES = (basestring, bool, dict, float, int, list, type(None)) -else: - LOG_RECORD_BUILT_IN_ATTRS.append('stack_info') - EASY_SERIALIZABLE_TYPES = (str, bool, dict, float, int, list, type(None)) +LOG_RECORD_BUILT_IN_ATTRS.append('stack_info') +EASY_SERIALIZABLE_TYPES = (str, bool, dict, float, int, list, type(None)) def _sanitize_log_msg(record): @@ -45,7 +42,7 @@ class BaseJSONFormatter(logging.Formatter): base_object_common = {} def __init__(self, *args, **kw): - super(BaseJSONFormatter, self).__init__(*args, **kw) + super().__init__(*args, **kw) if json_logging.COMPONENT_ID and json_logging.COMPONENT_ID != json_logging.EMPTY_VALUE: self.base_object_common["component_id"] = json_logging.COMPONENT_ID if json_logging.COMPONENT_NAME and json_logging.COMPONENT_NAME != json_logging.EMPTY_VALUE: @@ -120,7 +117,7 @@ def format_exception(cls, exc_info): return ''.join(traceback.format_exception(*exc_info)) if exc_info else '' def _format_log_object(self, record, request_util): - json_log_object = super(JSONLogFormatter, self)._format_log_object(record, request_util) + json_log_object = super()._format_log_object(record, request_util) json_log_object.update({ "msg": _sanitize_log_msg(record), @@ -144,7 +141,7 @@ class JSONLogWebFormatter(JSONLogFormatter): """ def _format_log_object(self, record, request_util): - json_log_object = super(JSONLogWebFormatter, self)._format_log_object(record, request_util) + json_log_object = super()._format_log_object(record, request_util) if json_logging.CORRELATION_ID_FIELD not in json_log_object: json_log_object.update({ @@ -160,7 +157,7 @@ class JSONRequestLogFormatter(BaseJSONFormatter): """ def _format_log_object(self, record, request_util): - json_log_object = super(JSONRequestLogFormatter, self)._format_log_object(record, request_util) + json_log_object = super()._format_log_object(record, request_util) request_adapter = request_util.request_adapter response_adapter = request_util.response_adapter diff --git a/json_logging/framework/__init__.py b/json_logging/framework/__init__.py index 9bad579..e69de29 100644 --- a/json_logging/framework/__init__.py +++ b/json_logging/framework/__init__.py @@ -1 +0,0 @@ -# coding=utf-8 diff --git a/json_logging/framework/connexion/__init__.py b/json_logging/framework/connexion/__init__.py index 9e4492e..7459092 100644 --- a/json_logging/framework/connexion/__init__.py +++ b/json_logging/framework/connexion/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 import logging import sys diff --git a/json_logging/framework/fastapi/__init__.py b/json_logging/framework/fastapi/__init__.py index 79cea21..76042bd 100644 --- a/json_logging/framework/fastapi/__init__.py +++ b/json_logging/framework/fastapi/__init__.py @@ -1,4 +1,3 @@ - def is_fastapi_present(): # noinspection PyPep8,PyBroadException try: diff --git a/json_logging/framework/flask/__init__.py b/json_logging/framework/flask/__init__.py index 03857ea..f9b29a8 100644 --- a/json_logging/framework/flask/__init__.py +++ b/json_logging/framework/flask/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 import logging import json_logging diff --git a/json_logging/framework/quart/__init__.py b/json_logging/framework/quart/__init__.py index 4043a70..3b05c8f 100644 --- a/json_logging/framework/quart/__init__.py +++ b/json_logging/framework/quart/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 import logging import sys diff --git a/json_logging/framework/sanic/__init__.py b/json_logging/framework/sanic/__init__.py index 9edba8b..4e4bea4 100644 --- a/json_logging/framework/sanic/__init__.py +++ b/json_logging/framework/sanic/__init__.py @@ -1,4 +1,3 @@ -# coding=utf-8 import logging import logging.config diff --git a/json_logging/framework_base.py b/json_logging/framework_base.py index 29bc3fc..df6eadf 100644 --- a/json_logging/framework_base.py +++ b/json_logging/framework_base.py @@ -1,4 +1,3 @@ -# coding=utf-8 class BaseRequestInfoExtractor: """ Helper class help to extract logging-relevant information from HTTP request object diff --git a/json_logging/util.py b/json_logging/util.py index c69d98a..a606ea4 100644 --- a/json_logging/util.py +++ b/json_logging/util.py @@ -1,4 +1,3 @@ -# coding=utf-8 import logging import os import re @@ -96,7 +95,7 @@ def currentframe(_no_of_go_up_level): return sys.exc_info()[_no_of_go_up_level - 1].tb_frame.f_back -class RequestUtil(object): +class RequestUtil: """ util for extract request's information """ diff --git a/setup.py b/setup.py index a4d2998..5c347ff 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,6 @@ -import io -import sys - from setuptools import setup, find_packages -version_info_major = sys.version_info[0] -if version_info_major == 3: - long_description = open('README.rst', encoding="utf8").read() -else: - io_open = io.open('README.rst', encoding="utf8") - long_description = io_open.read() +long_description = open('README.rst', encoding="utf8").read() setup( name="json-logging", @@ -21,7 +13,7 @@ keywords=["json", "elastic", "python", "python3", "python2", "logging", "logging-library", "json", "elasticsearch", "elk", "elk-stack", "logstash", "kibana"], platforms='any', - url="https://github.com/thangbn/json-logging", + url="https://github.com/virajkanwade/json-logging", classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', diff --git a/tests-performance/benmark_micro.py b/tests-performance/benmark_micro.py index 99ec8a9..22a3197 100644 --- a/tests-performance/benmark_micro.py +++ b/tests-performance/benmark_micro.py @@ -1,4 +1,3 @@ -# coding=utf-8 import time import timeit from datetime import datetime diff --git a/tests/helpers/handler.py b/tests/helpers/handler.py index 6b67d2e..e90acec 100644 --- a/tests/helpers/handler.py +++ b/tests/helpers/handler.py @@ -9,7 +9,7 @@ def __init__(self, level=logging.NOTSET) -> None: """Create a new log handler.""" super().__init__(level=level) self.level = level - self.messages: List[str] = [] + self.messages: list[str] = [] def emit(self, record: logging.LogRecord) -> None: """Keep the log records in a list in addition to the log text.""" From 41b9e9916bb9ea254fb67e7f81d6843134182002 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:23:40 -0700 Subject: [PATCH 2/8] Handle datetime.utcnow deprecation --- json_logging/dto.py | 6 +++--- json_logging/formatters.py | 10 ++-------- json_logging/util.py | 4 ++-- tests-performance/benmark_micro.py | 4 ++-- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/json_logging/dto.py b/json_logging/dto.py index 5d30041..0a48c68 100644 --- a/json_logging/dto.py +++ b/json_logging/dto.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from json_logging import util @@ -32,14 +32,14 @@ class DefaultRequestResponseDTO(RequestResponseDTOBase): def __init__(self, request, **kwargs): super().__init__(request, **kwargs) - utcnow = datetime.utcnow() + utcnow = datetime.now(timezone.utc) self._request_start = utcnow self["request_received_at"] = util.iso_time_format(utcnow) # noinspection PyAttributeOutsideInit def on_request_complete(self, response): super().on_request_complete(response) - utcnow = datetime.utcnow() + utcnow = datetime.now(timezone.utc) time_delta = utcnow - self._request_start self["response_time_ms"] = int(time_delta.total_seconds()) * 1000 + int(time_delta.microseconds / 1000) self["response_sent_at"] = util.iso_time_format(utcnow) diff --git a/json_logging/formatters.py b/json_logging/formatters.py index 62f784e..8fd652e 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -1,7 +1,7 @@ import logging import sys import traceback -from datetime import datetime +from datetime import datetime, timezone import json_logging @@ -16,12 +16,6 @@ 'props', ] -# python 2 compatible check -try: - basestring -except NameError: - basestring = str - LOG_RECORD_BUILT_IN_ATTRS.append('stack_info') EASY_SERIALIZABLE_TYPES = (str, bool, dict, float, int, list, type(None)) @@ -58,7 +52,7 @@ def format(self, record): return json_logging.JSON_SERIALIZER(log_object) def _format_log_object(self, record, request_util): - utcnow = datetime.utcnow() + utcnow = datetime.now(timezone.utc) base_obj = { "written_at": json_logging.util.iso_time_format(utcnow), diff --git a/json_logging/util.py b/json_logging/util.py index a606ea4..d2f7cf0 100644 --- a/json_logging/util.py +++ b/json_logging/util.py @@ -2,7 +2,7 @@ import os import re import sys -from datetime import datetime +from datetime import datetime, timezone from logging import Logger, StreamHandler import json_logging @@ -70,7 +70,7 @@ def validate_subclass(subclass, superclass): return True -_epoch = datetime(1970, 1, 1) +_epoch = datetime(1970, 1, 1, tzinfo=timezone.utc) def epoch_nano_second(datetime_): diff --git a/tests-performance/benmark_micro.py b/tests-performance/benmark_micro.py index 22a3197..4fb6438 100644 --- a/tests-performance/benmark_micro.py +++ b/tests-performance/benmark_micro.py @@ -1,8 +1,8 @@ import time import timeit -from datetime import datetime +from datetime import datetime, timezone -utcnow = datetime.utcnow() +utcnow = datetime.now(timezone.utc) numbers = 1000000 # timeit_timeit = timeit.timeit(lambda: '%04d-%02d-%02dT%02d:%02d:%02dZ' % ( From e48d0a5305fd650c365523d456e434ff92074db8 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:24:09 -0700 Subject: [PATCH 3/8] fix tests --- example/flask_sample_app.py | 4 +++- json_logging/framework/sanic/__init__.py | 2 -- tests/helpers/constants.py | 1 + tests/smoketests/fastapi/api.py | 4 +++- tests/smoketests/flask/api.py | 4 +++- tests/smoketests/quart/api.py | 4 +++- tests/smoketests/sanic/api.py | 4 +++- tests/smoketests/test_run_smoketest.py | 3 ++- 8 files changed, 18 insertions(+), 8 deletions(-) diff --git a/example/flask_sample_app.py b/example/flask_sample_app.py index 5da6012..e75673a 100644 --- a/example/flask_sample_app.py +++ b/example/flask_sample_app.py @@ -1,4 +1,5 @@ import logging +import os import sys import flask @@ -43,4 +44,5 @@ def exclude_from_request_instrumentation(): if __name__ == "__main__": - app.run(host='0.0.0.0', port=int(5000), use_reloader=False) + port = os.getenv('PORT', '5000') + app.run(host='0.0.0.0', port=int(port), use_reloader=False) diff --git a/json_logging/framework/sanic/__init__.py b/json_logging/framework/sanic/__init__.py index 4e4bea4..a00e29e 100644 --- a/json_logging/framework/sanic/__init__.py +++ b/json_logging/framework/sanic/__init__.py @@ -38,12 +38,10 @@ def config(self): LOGGING_CONFIG_DEFAULTS["formatters"]["generic"][ "class" ] = "json_logging.JSONLogFormatter" - del LOGGING_CONFIG_DEFAULTS["formatters"]["generic"]["format"] LOGGING_CONFIG_DEFAULTS["formatters"]["access"][ "class" ] = "json_logging.JSONLogFormatter" - del LOGGING_CONFIG_DEFAULTS["formatters"]["access"]["format"] # logging.config.dictConfig(LOGGING_CONFIG_DEFAULTS) diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index 9a355ba..3b6cfab 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -11,4 +11,5 @@ "module", "line_no", "correlation_id", + "taskName", } diff --git a/tests/smoketests/fastapi/api.py b/tests/smoketests/fastapi/api.py index dd6dbb7..7d57be2 100644 --- a/tests/smoketests/fastapi/api.py +++ b/tests/smoketests/fastapi/api.py @@ -1,3 +1,4 @@ +import os import datetime, logging, sys, json_logging, fastapi, uvicorn app = fastapi.FastAPI() @@ -17,4 +18,5 @@ def home(): return "Hello world : " + str(datetime.datetime.now()) if __name__ == "__main__": - uvicorn.run(app, host='0.0.0.0', port=5000) + port = os.getenv('PORT', '5000') + uvicorn.run(app, host='0.0.0.0', port=int(port)) diff --git a/tests/smoketests/flask/api.py b/tests/smoketests/flask/api.py index 89e5ff8..d4bfb0a 100644 --- a/tests/smoketests/flask/api.py +++ b/tests/smoketests/flask/api.py @@ -1,3 +1,4 @@ +import os import datetime, logging, sys, json_logging, flask app = flask.Flask(__name__) @@ -17,4 +18,5 @@ def home(): return "Hello world : " + str(datetime.datetime.now()) if __name__ == "__main__": - app.run(host='0.0.0.0', port=int(5000), use_reloader=False) + port = os.getenv('PORT', '5000') + app.run(host='0.0.0.0', port=int(port), use_reloader=False) diff --git a/tests/smoketests/quart/api.py b/tests/smoketests/quart/api.py index f7728fa..5cf09ef 100644 --- a/tests/smoketests/quart/api.py +++ b/tests/smoketests/quart/api.py @@ -1,3 +1,4 @@ +import os import asyncio, logging, sys, json_logging, quart app = quart.Quart(__name__) @@ -18,4 +19,5 @@ async def home(): if __name__ == "__main__": loop = asyncio.get_event_loop() - app.run(host='0.0.0.0', port=int(5000), use_reloader=False, loop=loop) + port = os.getenv('PORT', '5000') + app.run(host='0.0.0.0', port=int(port), use_reloader=False, loop=loop) diff --git a/tests/smoketests/sanic/api.py b/tests/smoketests/sanic/api.py index 7dffc62..d543f21 100644 --- a/tests/smoketests/sanic/api.py +++ b/tests/smoketests/sanic/api.py @@ -1,3 +1,4 @@ +import os import logging, sys, json_logging, sanic app = sanic.Sanic(name="sanic-web-app") @@ -21,4 +22,5 @@ async def home(request): return sanic.response.text("hello world") if __name__ == "__main__": - app.run(host="0.0.0.0", port=5000) + port = os.getenv('PORT', '5000') + app.run(host="0.0.0.0", port=int(port)) diff --git a/tests/smoketests/test_run_smoketest.py b/tests/smoketests/test_run_smoketest.py index 3933f7e..5e3842d 100644 --- a/tests/smoketests/test_run_smoketest.py +++ b/tests/smoketests/test_run_smoketest.py @@ -47,7 +47,8 @@ def test_api_example(backend): os.environ['NO_PROXY'] = 'localhost' os.environ['no_proxy'] = 'localhost' try: - response = requests.get("http://localhost:5000/", timeout=1) + port = os.getenv('PORT', '5000') + response = requests.get(f"http://localhost:{port}/", timeout=1) assert response.status_code == 200 return except requests.exceptions.Timeout: From ca1df71d16355fa026123106b3e4d62229e9d079 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:38:26 -0700 Subject: [PATCH 4/8] add check and format scripts --- .github/workflows/code_quality.yml | 4 ++-- pyproject.toml | 9 +++++++++ requirements-dev.txt | 2 ++ test-requirements.txt => requirements-test.txt | 0 setup.cfg | 6 +++++- 5 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt rename test-requirements.txt => requirements-test.txt (100%) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 9256df9..64b1dcf 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -21,7 +21,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test-requirements.txt + pip install -r requirements-test.txt - name: Flake8 run: | # stop the build if there are Python syntax errors or undefined names @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r test-requirements.txt + pip install -r requirements-test.txt - name: Run tests run: | pytest --ignore tests/smoketests diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..493643c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,9 @@ +[tool.ruff.lint.pyupgrade] +# Preserve types, even if a file imports `from __future__ import annotations`. +keep-runtime-typing = true + +[tool.pylint.format] +max-line-length = 119 + +[tool.ruff] +line-length = 119 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..cad961a --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pyupgrade +ruff diff --git a/test-requirements.txt b/requirements-test.txt similarity index 100% rename from test-requirements.txt rename to requirements-test.txt diff --git a/setup.cfg b/setup.cfg index 4b866a2..30e2254 100644 --- a/setup.cfg +++ b/setup.cfg @@ -6,4 +6,8 @@ universal=1 [metadata] license_file=LICENSE -long_description=file: README.rst \ No newline at end of file +long_description=file: README.rst + +[flake8] +max-line-length = 119 +extend-ignore = E203,E701 From c1fc21009b3b86303e0c50300f69d20c5ace993a Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:46:34 -0700 Subject: [PATCH 5/8] ruff formatting applied --- example/connexion-example/hello.py | 15 +-- example/custom_log_format.py | 11 +- example/custom_log_format_request.py | 32 ++--- example/fastapi_sample_app.py | 37 +++--- example/flask_sample_app.py | 22 ++-- example/non_web.py | 6 +- example/quart_sample_app.py | 13 +- example/sanic_sample_app.py | 17 +-- json_logging/__init__.py | 77 +++++++----- json_logging/dto.py | 6 +- json_logging/formatters.py | 117 +++++++++++------- json_logging/framework/connexion/__init__.py | 27 ++-- json_logging/framework/fastapi/__init__.py | 8 +- .../framework/fastapi/implementation.py | 39 +++--- json_logging/framework/flask/__init__.py | 31 +++-- json_logging/framework/quart/__init__.py | 39 +++--- json_logging/framework/sanic/__init__.py | 15 +-- json_logging/framework_base.py | 38 +++--- json_logging/frameworks.py | 91 +++++++++----- json_logging/util.py | 52 ++++---- tests-performance/benmark_micro.py | 10 +- .../flask_performance_test_pure_python.py | 29 +++-- .../flask_performance_test_v2.py | 33 +++-- tests/conftest.py | 3 +- tests/helpers/imports.py | 3 +- tests/smoketests/fastapi/api.py | 19 ++- tests/smoketests/flask/api.py | 18 ++- tests/smoketests/quart/api.py | 18 ++- tests/smoketests/sanic/api.py | 13 +- tests/smoketests/test_run_smoketest.py | 12 +- tests/test_fastapi.py | 5 +- tests/test_flask.py | 6 +- 32 files changed, 505 insertions(+), 357 deletions(-) diff --git a/example/connexion-example/hello.py b/example/connexion-example/hello.py index 09b9dec..034f433 100755 --- a/example/connexion-example/hello.py +++ b/example/connexion-example/hello.py @@ -3,6 +3,7 @@ import sys import connexion + import json_logging # init the logger as usual @@ -13,23 +14,23 @@ def post_greeting(name): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) - return f'Hello {name}' + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) + return f"Hello {name}" def exclude_from_request_instrumentation(name): - return f'Hello {name}. this request wont log request instrumentation information' + return f"Hello {name}. this request wont log request instrumentation information" def create(): - app = connexion.FlaskApp(__name__, port=9090, specification_dir='openapi/') + app = connexion.FlaskApp(__name__, port=9090, specification_dir="openapi/") json_logging.init_connexion(enable_json=True) - json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation']) + json_logging.init_request_instrument(app, exclude_url_patterns=[r"/exclude_from_request_instrumentation"]) - app.add_api('helloworld-api.yaml', arguments={'title': 'Hello World Example'}) + app.add_api("helloworld-api.yaml", arguments={"title": "Hello World Example"}) return app -if __name__ == '__main__': +if __name__ == "__main__": app = create() app.run() diff --git a/example/custom_log_format.py b/example/custom_log_format.py index 6c9d836..acbff06 100644 --- a/example/custom_log_format.py +++ b/example/custom_log_format.py @@ -8,8 +8,8 @@ def extra(**kw): - '''Add the required nested props layer''' - return {'extra': {'props': kw}} + """Add the required nested props layer""" + return {"extra": {"props": kw}} class CustomJSONLog(json_logging.formatters.JSONLogFormatter): @@ -18,10 +18,7 @@ class CustomJSONLog(json_logging.formatters.JSONLogFormatter): """ def format(self, record): - json_customized_log_object = ({ - "customized_prop": "customized value", - "message": record.getMessage() - }) + json_customized_log_object = {"customized_prop": "customized value", "message": record.getMessage()} return json.dumps(json_customized_log_object) @@ -35,4 +32,4 @@ def format(self, record): logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stderr)) -logger.info('sample log message') +logger.info("sample log message") diff --git a/example/custom_log_format_request.py b/example/custom_log_format_request.py index a766e20..1ec1b96 100644 --- a/example/custom_log_format_request.py +++ b/example/custom_log_format_request.py @@ -20,15 +20,17 @@ def _format_log_object(self, record, request_util): response = record.request_response_data._response json_log_object = super()._format_log_object(record, request_util) - json_log_object.update({ - "customized_prop": "customized value", - }) + json_log_object.update( + { + "customized_prop": "customized value", + } + ) return json_log_object class CustomDefaultRequestResponseDTO(json_logging.dto.DefaultRequestResponseDTO): """ - custom implementation + custom implementation """ def __init__(self, request, **kwargs): @@ -41,9 +43,12 @@ def on_request_complete(self, response): app = flask.Flask(__name__) json_logging.init_flask(enable_json=True) -json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'], - custom_formatter=CustomRequestJSONLog, - request_response_dto_class=CustomDefaultRequestResponseDTO) +json_logging.init_request_instrument( + app, + exclude_url_patterns=[r"/exclude_from_request_instrumentation"], + custom_formatter=CustomRequestJSONLog, + request_response_dto_class=CustomDefaultRequestResponseDTO, +) # init the logger as usual logger = logging.getLogger("test logger") @@ -51,16 +56,15 @@ def on_request_complete(self, response): logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.route('/') +@app.route("/") def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() - return "hello world" \ - "\ncorrelation_id : " + correlation_id + return f"hello world\ncorrelation_id : {correlation_id}" -@app.route('/exception') +@app.route("/exception") def exception(): try: raise RuntimeError @@ -70,10 +74,10 @@ def exception(): return "Error occurred, check log for detail" -@app.route('/exclude_from_request_instrumentation') +@app.route("/exclude_from_request_instrumentation") def exclude_from_request_instrumentation(): return "this request wont log request instrumentation information" if __name__ == "__main__": - app.run(host='0.0.0.0', port=int(5000), use_reloader=False) + app.run(host="0.0.0.0", port=int(5000), use_reloader=False) diff --git a/example/fastapi_sample_app.py b/example/fastapi_sample_app.py index 3909ebe..4865b4c 100644 --- a/example/fastapi_sample_app.py +++ b/example/fastapi_sample_app.py @@ -10,16 +10,16 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.DEBUG) -@app.get('/') + +@app.get("/") async def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() - return "hello world" \ - "\ncorrelation_id : " + correlation_id + return "hello world" "\ncorrelation_id : " + correlation_id -@app.get('/exception') +@app.get("/exception") def exception(): try: raise RuntimeError @@ -29,28 +29,29 @@ def exception(): return "Error occurred, check log for detail" -@app.get('/exclude_from_request_instrumentation') +@app.get("/exclude_from_request_instrumentation") def exclude_from_request_instrumentation(): return "this request wont log request instrumentation information" if __name__ == "__main__": import uvicorn + logging_config = { - 'version': 1, - 'disable_existing_loggers': False, - 'handlers': { - 'default_handler': { - 'class': 'logging.StreamHandler', - 'level': 'DEBUG', + "version": 1, + "disable_existing_loggers": False, + "handlers": { + "default_handler": { + "class": "logging.StreamHandler", + "level": "DEBUG", }, }, - 'loggers': { - '': { - 'handlers': ['default_handler'], + "loggers": { + "": { + "handlers": ["default_handler"], } - } + }, } json_logging.init_fastapi(enable_json=True) - json_logging.init_request_instrument(app, exclude_url_patterns=[r'^/exclude_from_request_instrumentation']) - uvicorn.run(app, host='0.0.0.0', port=5000, log_level="debug", log_config=logging_config) + json_logging.init_request_instrument(app, exclude_url_patterns=[r"^/exclude_from_request_instrumentation"]) + uvicorn.run(app, host="0.0.0.0", port=5000, log_level="debug", log_config=logging_config) diff --git a/example/flask_sample_app.py b/example/flask_sample_app.py index e75673a..aad87a7 100644 --- a/example/flask_sample_app.py +++ b/example/flask_sample_app.py @@ -8,7 +8,7 @@ app = flask.Flask(__name__) json_logging.init_flask(enable_json=True) -json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation']) +json_logging.init_request_instrument(app, exclude_url_patterns=[r"/exclude_from_request_instrumentation"]) # init the logger as usual logger = logging.getLogger("test logger") @@ -16,19 +16,19 @@ logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.route('/') +@app.route("/") def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) - logger.info("test log statement with custom correlation id", - extra={'props': {'correlation_id': 'custom_correlation_id'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) + logger.info( + "test log statement with custom correlation id", extra={"props": {"correlation_id": "custom_correlation_id"}} + ) correlation_id = json_logging.get_correlation_id() - return "hello world" \ - "\ncorrelation_id : " + correlation_id + return "hello world" "\ncorrelation_id : " + correlation_id -@app.route('/exception') +@app.route("/exception") def exception(): try: raise RuntimeError @@ -38,11 +38,11 @@ def exception(): return "Error occurred, check log for detail" -@app.route('/exclude_from_request_instrumentation') +@app.route("/exclude_from_request_instrumentation") def exclude_from_request_instrumentation(): return "this request wont log request instrumentation information" if __name__ == "__main__": - port = os.getenv('PORT', '5000') - app.run(host='0.0.0.0', port=int(port), use_reloader=False) + port = os.getenv("PORT", "5000") + app.run(host="0.0.0.0", port=int(port), use_reloader=False) diff --git a/example/non_web.py b/example/non_web.py index 3c0dc8e..22ea315 100644 --- a/example/non_web.py +++ b/example/non_web.py @@ -11,5 +11,7 @@ logger.addHandler(logging.StreamHandler(sys.stdout)) logger.info("test log statement") -logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) -logger.info("test log statement with custom correlation id", extra={'props': {'correlation_id': 'custom_correlation_id'}}) \ No newline at end of file +logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) +logger.info( + "test log statement with custom correlation id", extra={"props": {"correlation_id": "custom_correlation_id"}} +) diff --git a/example/quart_sample_app.py b/example/quart_sample_app.py index 6109c91..1acb1eb 100644 --- a/example/quart_sample_app.py +++ b/example/quart_sample_app.py @@ -8,7 +8,7 @@ app = quart.Quart(__name__) json_logging.init_quart(enable_json=True) -json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation']) +json_logging.init_request_instrument(app, exclude_url_patterns=[r"/exclude_from_request_instrumentation"]) # init the logger as usual logger = logging.getLogger("test logger") @@ -16,20 +16,19 @@ logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.route('/') +@app.route("/") async def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() - return "hello world" \ - "\ncorrelation_id : " + correlation_id + return "hello world" "\ncorrelation_id : " + correlation_id -@app.route('/exclude_from_request_instrumentation') +@app.route("/exclude_from_request_instrumentation") def exclude_from_request_instrumentation(): return "this request wont log request instrumentation information" if __name__ == "__main__": loop = asyncio.get_event_loop() - app.run(host='0.0.0.0', port=int(5001), use_reloader=False, loop=loop) + app.run(host="0.0.0.0", port=int(5001), use_reloader=False, loop=loop) diff --git a/example/sanic_sample_app.py b/example/sanic_sample_app.py index fbc1cb8..a95426c 100644 --- a/example/sanic_sample_app.py +++ b/example/sanic_sample_app.py @@ -1,14 +1,16 @@ #!/usr/bin/env python3 -import json_logging import logging +import sys + # noinspection PyPackageRequirements import sanic -import sys + +import json_logging app = sanic.Sanic(name="sanic-web-app") json_logging.init_sanic(enable_json=True) -json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation']) +json_logging.init_request_instrument(app, exclude_url_patterns=[r"/exclude_from_request_instrumentation"]) # init the logger as usual logger = logging.getLogger("sanic-integration-test-app") @@ -19,7 +21,7 @@ @app.route("/") def test(request): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) # this will be faster correlation_id = json_logging.get_correlation_id(request=request) # this will be slower, but will work in context you cant get a reference of request object @@ -27,11 +29,12 @@ def test(request): return sanic.response.text( "hello world" - "\ncorrelation_id : " + correlation_id + - "\ncorrelation_id_without_request_obj: " + correlation_id_without_request_obj) + f"\ncorrelation_id : {correlation_id}" + f"\ncorrelation_id_without_request_obj: {correlation_id_without_request_obj}" + ) -@app.route('/exclude_from_request_instrumentation') +@app.route("/exclude_from_request_instrumentation") def exclude_from_request_instrumentation(request): return sanic.response.text("this request wont log request instrumentation information") diff --git a/json_logging/__init__.py b/json_logging/__init__.py index c2a4d2d..fda4e55 100644 --- a/json_logging/__init__.py +++ b/json_logging/__init__.py @@ -4,24 +4,27 @@ import uuid from json_logging import util -from json_logging.dto import RequestResponseDTOBase, DefaultRequestResponseDTO -from json_logging.formatters import JSONRequestLogFormatter, JSONLogFormatter, JSONLogWebFormatter -from json_logging.framework_base import BaseRequestInfoExtractor, BaseResponseInfoExtractor, \ - BaseAppRequestInstrumentationConfigurator, \ - BaseFrameworkConfigurator +from json_logging.dto import DefaultRequestResponseDTO, RequestResponseDTOBase +from json_logging.formatters import JSONLogFormatter, JSONLogWebFormatter, JSONRequestLogFormatter +from json_logging.framework_base import ( + BaseAppRequestInstrumentationConfigurator, + BaseFrameworkConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, +) from json_logging.util import get_library_logger, is_env_var_toggle -CORRELATION_ID_FIELD = 'correlation_id' +CORRELATION_ID_FIELD = "correlation_id" CORRELATION_ID_GENERATOR = uuid.uuid1 ENABLE_JSON_LOGGING = False if is_env_var_toggle("ENABLE_JSON_LOGGING"): ENABLE_JSON_LOGGING = True ENABLE_JSON_LOGGING_DEBUG = False -EMPTY_VALUE = '-' +EMPTY_VALUE = "-" CREATE_CORRELATION_ID_IF_NOT_EXISTS = True JSON_SERIALIZER = lambda log: json.dumps(log, ensure_ascii=False, default=str) -CORRELATION_ID_HEADERS = ['X-Correlation-ID', 'X-Request-ID'] +CORRELATION_ID_HEADERS = ["X-Correlation-ID", "X-Request-ID"] COMPONENT_ID = EMPTY_VALUE COMPONENT_NAME = EMPTY_VALUE COMPONENT_INSTANCE_INDEX = 0 @@ -45,16 +48,17 @@ def get_correlation_id(request=None): def config_root_logger(): """ - You must call this if you are using root logger. - Make all root logger' handlers produce JSON format - & remove duplicate handlers for request instrumentation logging. - Please made sure that you call this after you called "logging.basicConfig() or logging.getLogger() + You must call this if you are using root logger. + Make all root logger' handlers produce JSON format + & remove duplicate handlers for request instrumentation logging. + Please made sure that you call this after you called "logging.basicConfig() or logging.getLogger() """ if not logging.root.handlers: _logger.error( "No logging handlers found for root logger. Please made sure that you call this after you called " - "logging.basicConfig() or logging.getLogger()") + "logging.basicConfig() or logging.getLogger()" + ) return if ENABLE_JSON_LOGGING: @@ -93,12 +97,13 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False): if custom_formatter: if not issubclass(custom_formatter, logging.Formatter): - raise ValueError('custom_formatter is not subclass of logging.Formatter', custom_formatter) + raise ValueError("custom_formatter is not subclass of logging.Formatter", custom_formatter) if not enable_json and not ENABLE_JSON_LOGGING: _logger.warning( "JSON format is not enabled, normal log will be in plain text but request logging still in JSON format! " - "To enable set ENABLE_JSON_LOGGING env var to either one of following values: ['true', '1', 'y', 'yes']") + "To enable set ENABLE_JSON_LOGGING env var to either one of following values: ['true', '1', 'y', 'yes']" + ) else: ENABLE_JSON_LOGGING = True @@ -113,11 +118,12 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False): _current_framework = _framework_support_map[framework_name] _request_util = util.RequestUtil( - request_info_extractor_class=_current_framework['request_info_extractor_class'], - response_info_extractor_class=_current_framework['response_info_extractor_class']) + request_info_extractor_class=_current_framework["request_info_extractor_class"], + response_info_extractor_class=_current_framework["response_info_extractor_class"], + ) - if ENABLE_JSON_LOGGING and _current_framework['app_configurator'] is not None: - _current_framework['app_configurator']().config() + if ENABLE_JSON_LOGGING and _current_framework["app_configurator"] is not None: + _current_framework["app_configurator"]().config() _default_formatter = custom_formatter if custom_formatter else JSONLogWebFormatter else: @@ -132,8 +138,9 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False): util.update_formatter_for_loggers(existing_loggers, _default_formatter) -def init_request_instrument(app=None, custom_formatter=None, exclude_url_patterns=[], - request_response_dto_class=DefaultRequestResponseDTO): +def init_request_instrument( + app=None, custom_formatter=None, exclude_url_patterns=[], request_response_dto_class=DefaultRequestResponseDTO +): """ Configure the request instrumentation logging configuration for given web app. Must be called after init method @@ -144,18 +151,19 @@ def init_request_instrument(app=None, custom_formatter=None, exclude_url_pattern :param request_response_dto_class: request_response_dto_class to override default json_logging.RequestResponseDataExtractor. """ - if _current_framework is None or _current_framework == '-': + if _current_framework is None or _current_framework == "-": raise RuntimeError("please init the logging first, call init(framework_name) first") if custom_formatter: if not issubclass(custom_formatter, logging.Formatter): - raise ValueError('custom_formatter is not subclass of logging.Formatter', custom_formatter) + raise ValueError("custom_formatter is not subclass of logging.Formatter", custom_formatter) if not issubclass(request_response_dto_class, RequestResponseDTOBase): - raise ValueError('request_response_dto_class is not subclass of json_logging.RequestInfoBase', - custom_formatter) + raise ValueError( + "request_response_dto_class is not subclass of json_logging.RequestInfoBase", custom_formatter + ) - configurator = _current_framework['app_request_instrumentation_configurator']() + configurator = _current_framework["app_request_instrumentation_configurator"]() configurator.config(app, request_response_dto_class, exclude_url_patterns=exclude_url_patterns) formatter = custom_formatter if custom_formatter else JSONRequestLogFormatter @@ -167,12 +175,13 @@ def init_request_instrument(app=None, custom_formatter=None, exclude_url_pattern def get_request_logger(): - if _current_framework is None or _current_framework == '-': + if _current_framework is None or _current_framework == "-": raise RuntimeError( "request_logger is only available if json_logging is inited with a web app, " - "call init_() to do that") + "call init_() to do that" + ) - instance = _current_framework['app_request_instrumentation_configurator']._instance + instance = _current_framework["app_request_instrumentation_configurator"]._instance if instance is None: raise RuntimeError("please init request instrument first, call init_request_instrument(app) to do that") @@ -183,20 +192,20 @@ def get_request_logger(): def init_flask(custom_formatter=None, enable_json=False): - __init(framework_name='flask', custom_formatter=custom_formatter, enable_json=enable_json) + __init(framework_name="flask", custom_formatter=custom_formatter, enable_json=enable_json) def init_sanic(custom_formatter=None, enable_json=False): - __init(framework_name='sanic', custom_formatter=custom_formatter, enable_json=enable_json) + __init(framework_name="sanic", custom_formatter=custom_formatter, enable_json=enable_json) def init_quart(custom_formatter=None, enable_json=False): - __init(framework_name='quart', custom_formatter=custom_formatter, enable_json=enable_json) + __init(framework_name="quart", custom_formatter=custom_formatter, enable_json=enable_json) def init_connexion(custom_formatter=None, enable_json=False): - __init(framework_name='connexion', custom_formatter=custom_formatter, enable_json=enable_json) + __init(framework_name="connexion", custom_formatter=custom_formatter, enable_json=enable_json) def init_fastapi(custom_formatter=None, enable_json=False): - __init(framework_name='fastapi', custom_formatter=custom_formatter, enable_json=enable_json) + __init(framework_name="fastapi", custom_formatter=custom_formatter, enable_json=enable_json) diff --git a/json_logging/dto.py b/json_logging/dto.py index 0a48c68..82c2004 100644 --- a/json_logging/dto.py +++ b/json_logging/dto.py @@ -5,8 +5,8 @@ class RequestResponseDTOBase(dict): """ - Data transfer object (DTO) for request instrumentation logging - Served as base class for any actual RequestResponseDTO implementation + Data transfer object (DTO) for request instrumentation logging + Served as base class for any actual RequestResponseDTO implementation """ def __init__(self, request, **kwargs): @@ -27,7 +27,7 @@ def on_request_complete(self, response): class DefaultRequestResponseDTO(RequestResponseDTOBase): """ - default implementation + default implementation """ def __init__(self, request, **kwargs): diff --git a/json_logging/formatters.py b/json_logging/formatters.py index 8fd652e..c990bd6 100644 --- a/json_logging/formatters.py +++ b/json_logging/formatters.py @@ -8,15 +8,35 @@ # The list contains all the attributes listed in that will not be overwritten by custom extra props # http://docs.python.org/library/logging.html#logrecord-attributes LOG_RECORD_BUILT_IN_ATTRS = [ - 'asctime', 'created', 'exc_info', 'exc_text', 'filename', 'args', - 'funcName', 'id', 'levelname', 'levelno', 'lineno', 'module', 'msg', - 'msecs', 'msecs', 'message', 'name', 'pathname', 'process', - 'processName', 'relativeCreated', 'thread', 'threadName', 'extra', + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "args", + "funcName", + "id", + "levelname", + "levelno", + "lineno", + "module", + "msg", + "msecs", + "msecs", + "message", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "thread", + "threadName", + "extra", # Also exclude legacy 'props' - 'props', + "props", ] -LOG_RECORD_BUILT_IN_ATTRS.append('stack_info') +LOG_RECORD_BUILT_IN_ATTRS.append("stack_info") EASY_SERIALIZABLE_TYPES = (str, bool, dict, float, int, list, type(None)) @@ -26,13 +46,14 @@ def _sanitize_log_msg(record): :param record: log object :return: sanitized log object """ - return record.getMessage().replace('\n', '_').replace('\r', '_').replace('\t', '_') + return record.getMessage().replace("\n", "_").replace("\r", "_").replace("\t", "_") class BaseJSONFormatter(logging.Formatter): """ - Base class for JSON formatters + Base class for JSON formatters """ + base_object_common = {} def __init__(self, *args, **kw): @@ -46,7 +67,7 @@ def __init__(self, *args, **kw): def format(self, record): """ - Format the specified record as text. Overriding default python logging implementation + Format the specified record as text. Overriding default python logging implementation """ log_object = self._format_log_object(record, request_util=json_logging._request_util) return json_logging.JSON_SERIALIZER(log_object) @@ -74,7 +95,7 @@ def _get_extra_fields(self, record): fields = {} if record.args: - fields['msg'] = record.msg + fields["msg"] = record.msg for key, value in record.__dict__.items(): if key not in LOG_RECORD_BUILT_IN_ATTRS: @@ -85,7 +106,7 @@ def _get_extra_fields(self, record): fields[key] = repr(value) # Always add 'props' to the root of the log, assumes props is a dict - if hasattr(record, 'props') and isinstance(record.props, dict): + if hasattr(record, "props") and isinstance(record.props, dict): fields.update(record.props) return fields @@ -102,26 +123,28 @@ def get_exc_fields(self, record): else: exc_info = record.exc_text return { - 'exc_info': exc_info, - 'filename': record.filename, + "exc_info": exc_info, + "filename": record.filename, } @classmethod def format_exception(cls, exc_info): - return ''.join(traceback.format_exception(*exc_info)) if exc_info else '' + return "".join(traceback.format_exception(*exc_info)) if exc_info else "" def _format_log_object(self, record, request_util): json_log_object = super()._format_log_object(record, request_util) - json_log_object.update({ - "msg": _sanitize_log_msg(record), - "type": "log", - "logger": record.name, - "thread": record.threadName, - "level": record.levelname, - "module": record.module, - "line_no": record.lineno, - }) + json_log_object.update( + { + "msg": _sanitize_log_msg(record), + "type": "log", + "logger": record.name, + "thread": record.threadName, + "level": record.levelname, + "module": record.module, + "line_no": record.lineno, + } + ) if record.exc_info or record.exc_text: json_log_object.update(self.get_exc_fields(record)) @@ -138,16 +161,18 @@ def _format_log_object(self, record, request_util): json_log_object = super()._format_log_object(record, request_util) if json_logging.CORRELATION_ID_FIELD not in json_log_object: - json_log_object.update({ - json_logging.CORRELATION_ID_FIELD: request_util.get_correlation_id(within_formatter=True), - }) + json_log_object.update( + { + json_logging.CORRELATION_ID_FIELD: request_util.get_correlation_id(within_formatter=True), + } + ) return json_log_object class JSONRequestLogFormatter(BaseJSONFormatter): """ - Formatter for HTTP request instrumentation logging + Formatter for HTTP request instrumentation logging """ def _format_log_object(self, record, request_util): @@ -161,23 +186,27 @@ def _format_log_object(self, record, request_util): length = request_adapter.get_content_length(request) - json_log_object.update({ - "type": "request", - json_logging.CORRELATION_ID_FIELD: request_util.get_correlation_id(request), - "remote_user": request_adapter.get_remote_user(request), - "request": request_adapter.get_path(request), - "referer": request_adapter.get_http_header(request, 'referer', json_logging.EMPTY_VALUE), - "x_forwarded_for": request_adapter.get_http_header(request, 'x-forwarded-for', json_logging.EMPTY_VALUE), - "protocol": request_adapter.get_protocol(request), - "method": request_adapter.get_method(request), - "remote_ip": request_adapter.get_remote_ip(request), - "request_size_b": json_logging.util.parse_int(length, -1), - "remote_host": request_adapter.get_remote_ip(request), - "remote_port": request_adapter.get_remote_port(request), - "response_status": response_adapter.get_status_code(response), - "response_size_b": response_adapter.get_response_size(response), - "response_content_type": response_adapter.get_content_type(response), - }) + json_log_object.update( + { + "type": "request", + json_logging.CORRELATION_ID_FIELD: request_util.get_correlation_id(request), + "remote_user": request_adapter.get_remote_user(request), + "request": request_adapter.get_path(request), + "referer": request_adapter.get_http_header(request, "referer", json_logging.EMPTY_VALUE), + "x_forwarded_for": request_adapter.get_http_header( + request, "x-forwarded-for", json_logging.EMPTY_VALUE + ), + "protocol": request_adapter.get_protocol(request), + "method": request_adapter.get_method(request), + "remote_ip": request_adapter.get_remote_ip(request), + "request_size_b": json_logging.util.parse_int(length, -1), + "remote_host": request_adapter.get_remote_ip(request), + "remote_port": request_adapter.get_remote_port(request), + "response_status": response_adapter.get_status_code(response), + "response_size_b": response_adapter.get_response_size(response), + "response_content_type": response_adapter.get_content_type(response), + } + ) json_log_object.update(record.request_response_data) diff --git a/json_logging/framework/connexion/__init__.py b/json_logging/framework/connexion/__init__.py index 7459092..13c62fe 100644 --- a/json_logging/framework/connexion/__init__.py +++ b/json_logging/framework/connexion/__init__.py @@ -4,8 +4,11 @@ import json_logging import json_logging.framework from json_logging import JSONLogWebFormatter -from json_logging.framework_base import BaseAppRequestInstrumentationConfigurator, BaseRequestInfoExtractor, \ - BaseResponseInfoExtractor +from json_logging.framework_base import ( + BaseAppRequestInstrumentationConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, +) from json_logging.util import is_not_match_any_pattern @@ -13,15 +16,16 @@ def is_connexion_present(): # noinspection PyPep8,PyBroadException try: import connexion + return True except: return False if is_connexion_present(): - from connexion import request as request_obj import connexion as connexion import flask as flask + from connexion import request as request_obj from flask import g _current_request = request_obj @@ -35,16 +39,17 @@ def config(self, app, request_response_dto_class, exclude_url_patterns=[]): if not is_connexion_present(): raise RuntimeError("connexion is not available in system runtime") from flask.app import Flask + if not isinstance(app.app, Flask): raise RuntimeError("app is not a valid connexion.app.Connexion app instance") # Disable standard logging - logging.getLogger('werkzeug').disabled = True + logging.getLogger("werkzeug").disabled = True - json_logging.util.update_formatter_for_loggers([logging.getLogger('werkzeug')], JSONLogWebFormatter) + json_logging.util.update_formatter_for_loggers([logging.getLogger("werkzeug")], JSONLogWebFormatter) # noinspection PyAttributeOutsideInit - self.request_logger = logging.getLogger('connexion-request-logger') + self.request_logger = logging.getLogger("connexion-request-logger") from flask import g @@ -55,10 +60,10 @@ def before_request(): @app.app.after_request def after_request(response): - if hasattr(g, 'request_response_data'): + if hasattr(g, "request_response_data"): request_response_data = g.request_response_data request_response_data.on_request_complete(response) - self.request_logger.info("", extra={'request_response_data': request_response_data}) + self.request_logger.info("", extra={"request_response_data": request_response_data}) return response @@ -97,12 +102,12 @@ def set_correlation_id(self, request_, value): def get_correlation_id_in_request_context(self, request): try: - return _connexion.g.get('correlation_id', None) + return _connexion.g.get("correlation_id", None) except: return None def get_protocol(self, request): - return request.environ.get('SERVER_PROTOCOL') + return request.environ.get("SERVER_PROTOCOL") def get_path(self, request): return request.path @@ -117,7 +122,7 @@ def get_remote_ip(self, request): return request.remote_addr def get_remote_port(self, request): - return request.environ.get('REMOTE_PORT') + return request.environ.get("REMOTE_PORT") class ConnexionResponseInfoExtractor(BaseResponseInfoExtractor): diff --git a/json_logging/framework/fastapi/__init__.py b/json_logging/framework/fastapi/__init__.py index 76042bd..bce5d8f 100644 --- a/json_logging/framework/fastapi/__init__.py +++ b/json_logging/framework/fastapi/__init__.py @@ -3,11 +3,15 @@ def is_fastapi_present(): try: import fastapi import starlette + return True except: return False if is_fastapi_present(): - from .implementation import FastAPIAppRequestInstrumentationConfigurator, FastAPIRequestInfoExtractor, \ - FastAPIResponseInfoExtractor + from .implementation import ( + FastAPIAppRequestInstrumentationConfigurator, + FastAPIRequestInfoExtractor, + FastAPIResponseInfoExtractor, + ) diff --git a/json_logging/framework/fastapi/implementation.py b/json_logging/framework/fastapi/implementation.py index 71bd85e..b9fa744 100644 --- a/json_logging/framework/fastapi/implementation.py +++ b/json_logging/framework/fastapi/implementation.py @@ -1,28 +1,29 @@ import logging -import json_logging -import json_logging.framework -from json_logging.framework_base import BaseAppRequestInstrumentationConfigurator, BaseRequestInfoExtractor, \ - BaseResponseInfoExtractor - -from json_logging.util import is_not_match_any_pattern - import fastapi import starlette.requests import starlette.responses - from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint from starlette.requests import Request from starlette.responses import Response from starlette.types import ASGIApp +import json_logging +import json_logging.framework +from json_logging.framework_base import ( + BaseAppRequestInstrumentationConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, +) +from json_logging.util import is_not_match_any_pattern + _request_config_class = None class JSONLoggingASGIMiddleware(BaseHTTPMiddleware): def __init__(self, app: ASGIApp, exclude_url_patterns=tuple()) -> None: super().__init__(app) - self.request_logger = logging.getLogger('fastapi-request-logger') + self.request_logger = logging.getLogger("fastapi-request-logger") self.exclude_url_patterns = exclude_url_patterns logging.getLogger("uvicorn.access").propagate = False @@ -35,9 +36,7 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) - request_response_data = _request_config_class(request) response = await call_next(request) request_response_data.on_request_complete(response) - self.request_logger.info( - "", extra={"request_response_data": request_response_data, "type": "request"} - ) + self.request_logger.info("", extra={"request_response_data": request_response_data, "type": "request"}) return response @@ -50,10 +49,10 @@ def config(self, app, request_response_dto_class, exclude_url_patterns=[]): _request_config_class = request_response_dto_class # Disable standard logging - logging.getLogger('uvicorn.access').disabled = True + logging.getLogger("uvicorn.access").disabled = True # noinspection PyAttributeOutsideInit - self.request_logger = logging.getLogger('fastapi-request-logger') + self.request_logger = logging.getLogger("fastapi-request-logger") app.add_middleware(JSONLoggingASGIMiddleware, exclude_url_patterns=exclude_url_patterns) @@ -95,9 +94,9 @@ def get_correlation_id_in_request_context(self, request: starlette.requests.Requ return None def get_protocol(self, request: starlette.requests.Request): - protocol = str(request.scope.get('type', '')) - http_version = str(request.scope.get('http_version', '')) - if protocol.lower() == 'http' and http_version: + protocol = str(request.scope.get("type", "")) + http_version = str(request.scope.get("http_version", "")) + if protocol.lower() == "http" and http_version: return protocol.upper() + "/" + http_version return json_logging.EMPTY_VALUE @@ -105,7 +104,7 @@ def get_path(self, request: starlette.requests.Request): return request.url.path def get_content_length(self, request: starlette.requests.Request): - return request.headers.get('content-length', json_logging.EMPTY_VALUE) + return request.headers.get("content-length", json_logging.EMPTY_VALUE) def get_method(self, request: starlette.requests.Request): return request.method @@ -122,7 +121,7 @@ def get_status_code(self, response: starlette.responses.Response): return response.status_code def get_response_size(self, response: starlette.responses.Response): - return response.headers.get('content-length', json_logging.EMPTY_VALUE) + return response.headers.get("content-length", json_logging.EMPTY_VALUE) def get_content_type(self, response: starlette.responses.Response): - return response.headers.get('content-type', json_logging.EMPTY_VALUE) + return response.headers.get("content-type", json_logging.EMPTY_VALUE) diff --git a/json_logging/framework/flask/__init__.py b/json_logging/framework/flask/__init__.py index f9b29a8..3b1c60f 100644 --- a/json_logging/framework/flask/__init__.py +++ b/json_logging/framework/flask/__init__.py @@ -3,9 +3,11 @@ import json_logging import json_logging.formatters import json_logging.framework -from json_logging.framework_base import BaseAppRequestInstrumentationConfigurator, BaseRequestInfoExtractor, \ - BaseResponseInfoExtractor - +from json_logging.framework_base import ( + BaseAppRequestInstrumentationConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, +) from json_logging.util import is_not_match_any_pattern @@ -13,14 +15,15 @@ def is_flask_present(): # noinspection PyPep8,PyBroadException try: import flask + return True except: return False if is_flask_present(): - from flask import request as request_obj import flask as flask + from flask import request as request_obj _current_request = request_obj _flask = flask @@ -32,17 +35,19 @@ def config(self, app, request_response_dto_class, exclude_url_patterns=[]): raise RuntimeError("flask is not available in system runtime") from flask.app import Flask + if not isinstance(app, Flask): raise RuntimeError("app is not a valid flask.app.Flask app instance") # Disable standard logging - logging.getLogger('werkzeug').disabled = True + logging.getLogger("werkzeug").disabled = True - json_logging.util.update_formatter_for_loggers([logging.getLogger('werkzeug')], - json_logging.formatters.JSONLogWebFormatter) + json_logging.util.update_formatter_for_loggers( + [logging.getLogger("werkzeug")], json_logging.formatters.JSONLogWebFormatter + ) # noinspection PyAttributeOutsideInit - self.request_logger = logging.getLogger('flask-request-logger') + self.request_logger = logging.getLogger("flask-request-logger") from flask import g @@ -53,10 +58,10 @@ def before_request(): @app.after_request def after_request(response): - if hasattr(g, 'request_response_data'): + if hasattr(g, "request_response_data"): request_response_data = g.request_response_data request_response_data.on_request_complete(response) - self.request_logger.info("", extra={'request_response_data': request_response_data}) + self.request_logger.info("", extra={"request_response_data": request_response_data}) return response @@ -96,12 +101,12 @@ def set_correlation_id(self, request_, value): def get_correlation_id_in_request_context(self, request): try: - return _flask.g.get('correlation_id', None) + return _flask.g.get("correlation_id", None) except: return None def get_protocol(self, request): - return request.environ.get('SERVER_PROTOCOL') + return request.environ.get("SERVER_PROTOCOL") def get_path(self, request): return request.path @@ -116,7 +121,7 @@ def get_remote_ip(self, request): return request.remote_addr def get_remote_port(self, request): - return request.environ.get('REMOTE_PORT') + return request.environ.get("REMOTE_PORT") class FlaskResponseInfoExtractor(BaseResponseInfoExtractor): diff --git a/json_logging/framework/quart/__init__.py b/json_logging/framework/quart/__init__.py index 3b05c8f..e756e3a 100644 --- a/json_logging/framework/quart/__init__.py +++ b/json_logging/framework/quart/__init__.py @@ -4,8 +4,11 @@ import json_logging import json_logging.framework from json_logging import JSONLogWebFormatter -from json_logging.framework_base import BaseAppRequestInstrumentationConfigurator, BaseRequestInfoExtractor, \ - BaseResponseInfoExtractor +from json_logging.framework_base import ( + BaseAppRequestInstrumentationConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, +) from json_logging.util import is_not_match_any_pattern @@ -13,14 +16,15 @@ def is_quart_present(): # noinspection PyPep8,PyBroadException try: from quart import Quart + return True except: return False if is_quart_present(): - from quart import request as request_obj import quart as quart + from quart import request as request_obj _current_request = request_obj _quart = quart @@ -31,20 +35,25 @@ def config(self, app, request_response_dto_class, exclude_url_patterns=[]): if not is_quart_present(): raise RuntimeError("quart is not available in system runtime") from quart.app import Quart + if not isinstance(app, Quart): raise RuntimeError("app is not a valid quart.app.Quart app instance") # Remove quart logging handlers from quart.logging import default_handler - logging.getLogger('quart.app').removeHandler(default_handler) - json_logging.util.update_formatter_for_loggers([ - logging.getLogger('quart.app'), - ], JSONLogWebFormatter) - logging.getLogger('quart.serving').disabled = True + logging.getLogger("quart.app").removeHandler(default_handler) + json_logging.util.update_formatter_for_loggers( + [ + logging.getLogger("quart.app"), + ], + JSONLogWebFormatter, + ) + + logging.getLogger("quart.serving").disabled = True # noinspection PyAttributeOutsideInit - self.request_logger = logging.getLogger('quart-request-logger') + self.request_logger = logging.getLogger("quart-request-logger") from quart import g @@ -55,11 +64,11 @@ def before_request(): @app.after_request def after_request(response): - if hasattr(g, 'request_response_data'): + if hasattr(g, "request_response_data"): request_response_data = g.request_response_data request_response_data.on_request_complete(response) # TODO:handle to print out request instrumentation in non-JSON mode - self.request_logger.info("", extra={'request_response_data': request_response_data}) + self.request_logger.info("", extra={"request_response_data": request_response_data}) return response @@ -98,7 +107,7 @@ def set_correlation_id(self, request_, value): def get_correlation_id_in_request_context(self, request): try: - return _quart.g.get('correlation_id', None) + return _quart.g.get("correlation_id", None) except: return None @@ -118,11 +127,7 @@ def get_remote_ip(self, request): return request.remote_addr def get_remote_port(self, request): - return ( - request.host.split(":", 2)[1] - if len(request.host.split(":", 2)) == 2 - else None - ) + return request.host.split(":", 2)[1] if len(request.host.split(":", 2)) == 2 else None class QuartResponseInfoExtractor(BaseResponseInfoExtractor): diff --git a/json_logging/framework/sanic/__init__.py b/json_logging/framework/sanic/__init__.py index a00e29e..294e218 100644 --- a/json_logging/framework/sanic/__init__.py +++ b/json_logging/framework/sanic/__init__.py @@ -1,13 +1,12 @@ import logging import logging.config - import sys import json_logging import json_logging.framework from json_logging.framework_base import ( - BaseFrameworkConfigurator, BaseAppRequestInstrumentationConfigurator, + BaseFrameworkConfigurator, BaseRequestInfoExtractor, BaseResponseInfoExtractor, ) @@ -35,13 +34,9 @@ def config(self): from sanic.log import LOGGING_CONFIG_DEFAULTS LOGGING_CONFIG_DEFAULTS["disable_existing_loggers"] = False - LOGGING_CONFIG_DEFAULTS["formatters"]["generic"][ - "class" - ] = "json_logging.JSONLogFormatter" + LOGGING_CONFIG_DEFAULTS["formatters"]["generic"]["class"] = "json_logging.JSONLogFormatter" - LOGGING_CONFIG_DEFAULTS["formatters"]["access"][ - "class" - ] = "json_logging.JSONLogFormatter" + LOGGING_CONFIG_DEFAULTS["formatters"]["access"]["class"] = "json_logging.JSONLogFormatter" # logging.config.dictConfig(LOGGING_CONFIG_DEFAULTS) @@ -71,9 +66,7 @@ def after_request(request, response): if hasattr(request.ctx, "request_response_data"): request_response_data = request.ctx.request_response_data request_response_data.on_request_complete(response) - self.request_logger.info( - "", extra={"request_response_data": request_response_data, "type": "request"} - ) + self.request_logger.info("", extra={"request_response_data": request_response_data, "type": "request"}) class SanicRequestInfoExtractor(BaseRequestInfoExtractor): diff --git a/json_logging/framework_base.py b/json_logging/framework_base.py index df6eadf..1a2bc6f 100644 --- a/json_logging/framework_base.py +++ b/json_logging/framework_base.py @@ -1,31 +1,31 @@ class BaseRequestInfoExtractor: """ - Helper class help to extract logging-relevant information from HTTP request object + Helper class help to extract logging-relevant information from HTTP request object """ def __new__(cls, *arg, **kwargs): - if not hasattr(cls, '_instance'): + if not hasattr(cls, "_instance"): cls._instance = object.__new__(cls) return cls._instance @staticmethod def support_global_request_object(): """ - whether current framework supports global request object like Flask + whether current framework supports global request object like Flask """ raise NotImplementedError @staticmethod def get_current_request(): """ - get current request, should only implement for framework that support global request object + get current request, should only implement for framework that support global request object """ raise NotImplementedError @staticmethod def get_request_class_type(): """ - class type of request object, only need to specify in case the framework dont support global request object + class type of request object, only need to specify in case the framework dont support global request object """ raise NotImplementedError @@ -77,10 +77,10 @@ def get_protocol(self, request): def get_path(self, request): """ - We can safely assume that request is valid request object.\n - Gets the request path. + We can safely assume that request is valid request object.\n + Gets the request path. - :return: the request path (e.g. /index.html) + :return: the request path (e.g. /index.html) """ raise NotImplementedError @@ -123,11 +123,11 @@ def get_remote_port(self, request): class BaseResponseInfoExtractor: """ - Helper class help to extract logging-relevant information from HTTP response object + Helper class help to extract logging-relevant information from HTTP response object """ def __new__(cls, *arg, **kwargs): - if not hasattr(cls, '_instance'): + if not hasattr(cls, "_instance"): cls._instance = object.__new__(cls) return cls._instance @@ -158,32 +158,32 @@ def get_content_type(self, response): class BaseFrameworkConfigurator: """ - Class to perform logging configuration for given framework as needed, like disable built in request logging and other utils logging + Class to perform logging configuration for given framework as needed, like disable built in request logging and other utils logging """ def __new__(cls, *args, **kw): - if not hasattr(cls, '_instance'): + if not hasattr(cls, "_instance"): cls._instance = object.__new__(cls) return cls._instance def config(self): """ - app logging configuration logic + app logging configuration logic """ raise NotImplementedError class BaseAppRequestInstrumentationConfigurator: """ - Class to perform request instrumentation logging configuration. Should at least contains: - 1- register before-request hook and create a RequestInfo object, store it to request context - 2- register after-request hook and update response to stored RequestInfo object - 3 - re-configure framework loggers. - NOTE: logger that is used to emit request instrumentation logs will need to assign to **self.request_logger** + Class to perform request instrumentation logging configuration. Should at least contains: + 1- register before-request hook and create a RequestInfo object, store it to request context + 2- register after-request hook and update response to stored RequestInfo object + 3 - re-configure framework loggers. + NOTE: logger that is used to emit request instrumentation logs will need to assign to **self.request_logger** """ def __new__(cls, *args, **kw): - if not hasattr(cls, '_instance'): + if not hasattr(cls, "_instance"): cls._instance = object.__new__(cls) cls._instance.request_logger = None return cls._instance diff --git a/json_logging/frameworks.py b/json_logging/frameworks.py index a4d9d44..9b1b481 100644 --- a/json_logging/frameworks.py +++ b/json_logging/frameworks.py @@ -1,14 +1,25 @@ # register flask support # noinspection PyPep8 import json_logging.framework.flask as flask_support -from json_logging import util, BaseRequestInfoExtractor, BaseResponseInfoExtractor, \ - BaseAppRequestInstrumentationConfigurator, \ - BaseFrameworkConfigurator, _framework_support_map, ENABLE_JSON_LOGGING_DEBUG, _logger +from json_logging import ( + ENABLE_JSON_LOGGING_DEBUG, + BaseAppRequestInstrumentationConfigurator, + BaseFrameworkConfigurator, + BaseRequestInfoExtractor, + BaseResponseInfoExtractor, + _framework_support_map, + _logger, + util, +) -def register_framework_support(name, app_configurator, app_request_instrumentation_configurator, - request_info_extractor_class, - response_info_extractor_class): +def register_framework_support( + name, + app_configurator, + app_request_instrumentation_configurator, + request_info_extractor_class, + response_info_extractor_class, +): """ register support for a framework @@ -31,48 +42,70 @@ def register_framework_support(name, app_configurator, app_request_instrumentati if name in _framework_support_map: ENABLE_JSON_LOGGING_DEBUG and _logger.warning("Re-register framework %s", name) _framework_support_map[name] = { - 'app_configurator': app_configurator, - 'app_request_instrumentation_configurator': app_request_instrumentation_configurator, - 'request_info_extractor_class': request_info_extractor_class, - 'response_info_extractor_class': response_info_extractor_class + "app_configurator": app_configurator, + "app_request_instrumentation_configurator": app_request_instrumentation_configurator, + "request_info_extractor_class": request_info_extractor_class, + "response_info_extractor_class": response_info_extractor_class, } -register_framework_support('flask', None, flask_support.FlaskAppRequestInstrumentationConfigurator, - flask_support.FlaskRequestInfoExtractor, - flask_support.FlaskResponseInfoExtractor) +register_framework_support( + "flask", + None, + flask_support.FlaskAppRequestInstrumentationConfigurator, + flask_support.FlaskRequestInfoExtractor, + flask_support.FlaskResponseInfoExtractor, +) # register sanic support # noinspection PyPep8 -from json_logging.framework.sanic import SanicAppConfigurator, SanicAppRequestInstrumentationConfigurator, \ - SanicRequestInfoExtractor, SanicResponseInfoExtractor +from json_logging.framework.sanic import ( + SanicAppConfigurator, + SanicAppRequestInstrumentationConfigurator, + SanicRequestInfoExtractor, + SanicResponseInfoExtractor, +) -register_framework_support('sanic', SanicAppConfigurator, - SanicAppRequestInstrumentationConfigurator, - SanicRequestInfoExtractor, - SanicResponseInfoExtractor) +register_framework_support( + "sanic", + SanicAppConfigurator, + SanicAppRequestInstrumentationConfigurator, + SanicRequestInfoExtractor, + SanicResponseInfoExtractor, +) # register quart support # noinspection PyPep8 import json_logging.framework.quart as quart_support -register_framework_support('quart', None, quart_support.QuartAppRequestInstrumentationConfigurator, - quart_support.QuartRequestInfoExtractor, - quart_support.QuartResponseInfoExtractor) +register_framework_support( + "quart", + None, + quart_support.QuartAppRequestInstrumentationConfigurator, + quart_support.QuartRequestInfoExtractor, + quart_support.QuartResponseInfoExtractor, +) # register connexion support # noinspection PyPep8 import json_logging.framework.connexion as connexion_support -register_framework_support('connexion', None, connexion_support.ConnexionAppRequestInstrumentationConfigurator, - connexion_support.ConnexionRequestInfoExtractor, - connexion_support.ConnexionResponseInfoExtractor) +register_framework_support( + "connexion", + None, + connexion_support.ConnexionAppRequestInstrumentationConfigurator, + connexion_support.ConnexionRequestInfoExtractor, + connexion_support.ConnexionResponseInfoExtractor, +) # register FastAPI support import json_logging.framework.fastapi as fastapi_support if fastapi_support.is_fastapi_present(): - register_framework_support('fastapi', app_configurator=None, - app_request_instrumentation_configurator=fastapi_support.FastAPIAppRequestInstrumentationConfigurator, - request_info_extractor_class=fastapi_support.FastAPIRequestInfoExtractor, - response_info_extractor_class=fastapi_support.FastAPIResponseInfoExtractor) + register_framework_support( + "fastapi", + app_configurator=None, + app_request_instrumentation_configurator=fastapi_support.FastAPIAppRequestInstrumentationConfigurator, + request_info_extractor_class=fastapi_support.FastAPIRequestInfoExtractor, + response_info_extractor_class=fastapi_support.FastAPIResponseInfoExtractor, + ) diff --git a/json_logging/util.py b/json_logging/util.py index d2f7cf0..ff25a79 100644 --- a/json_logging/util.py +++ b/json_logging/util.py @@ -1,3 +1,4 @@ +import inspect import logging import os import re @@ -6,12 +7,11 @@ from logging import Logger, StreamHandler import json_logging -import inspect def is_env_var_toggle(var_name): enable_json_setting = str(os.getenv(var_name)).lower() - _env_toggle = enable_json_setting.lower() in ['true', '1', 'y', 'yes'] + _env_toggle = enable_json_setting.lower() in ["true", "1", "y", "yes"] return _env_toggle @@ -65,7 +65,7 @@ def validate_subclass(subclass, superclass): :return: bool """ if not issubclass(subclass, superclass): - raise RuntimeError(str(subclass) + ' is not a subclass of ' + str(superclass)) + raise RuntimeError(str(subclass) + " is not a subclass of " + str(superclass)) return True @@ -78,12 +78,18 @@ def epoch_nano_second(datetime_): def iso_time_format(datetime_): - return '%04d-%02d-%02dT%02d:%02d:%02d.%03dZ' % ( - datetime_.year, datetime_.month, datetime_.day, datetime_.hour, datetime_.minute, datetime_.second, - int(datetime_.microsecond / 1000)) - - -if hasattr(sys, '_getframe'): + return "%04d-%02d-%02dT%02d:%02d:%02d.%03dZ" % ( + datetime_.year, + datetime_.month, + datetime_.day, + datetime_.hour, + datetime_.minute, + datetime_.second, + int(datetime_.microsecond / 1000), + ) + + +if hasattr(sys, "_getframe"): currentframe = lambda _no_of_go_up_level: sys._getframe(_no_of_go_up_level) else: # pragma: no cover # noinspection PyBroadException @@ -97,14 +103,14 @@ def currentframe(_no_of_go_up_level): class RequestUtil: """ - util for extract request's information + util for extract request's information """ def __new__(cls, *args, **kw): # make this request util a singleton object - if not hasattr(cls, '_instance'): - request_info_extractor_class = kw['request_info_extractor_class'] - response_info_extractor_class = kw['response_info_extractor_class'] + if not hasattr(cls, "_instance"): + request_info_extractor_class = kw["request_info_extractor_class"] + response_info_extractor_class = kw["response_info_extractor_class"] validate_subclass(request_info_extractor_class, json_logging.BaseRequestInfoExtractor) validate_subclass(response_info_extractor_class, json_logging.BaseResponseInfoExtractor) @@ -115,7 +121,9 @@ def __new__(cls, *args, **kw): cls._instance.response_info_extractor_class = response_info_extractor_class cls._instance.request_adapter = request_info_extractor_class() cls._instance.response_adapter = response_info_extractor_class() - cls._instance.is_support_global_request_object = request_info_extractor_class.support_global_request_object() + cls._instance.is_support_global_request_object = ( + request_info_extractor_class.support_global_request_object() + ) cls._instance.create_correlation_id_if_not_exists = json_logging.CREATE_CORRELATION_ID_IF_NOT_EXISTS return cls._instance @@ -178,16 +186,16 @@ def get_request_from_call_stack(self, within_formatter=False): f = currentframe(no_of_go_up_level) while True: f_locals = f.f_locals - if 'request' in f_locals: - if isinstance(f_locals['request'], class_type): - return f_locals['request'] + if "request" in f_locals: + if isinstance(f_locals["request"], class_type): + return f_locals["request"] - if 'req' in f_locals: - if isinstance(f_locals['req'], class_type): - return f_locals['req'] + if "req" in f_locals: + if isinstance(f_locals["req"], class_type): + return f_locals["req"] for key in f_locals: - if key not in {'request', 'req'} and isinstance(f_locals[key], class_type): + if key not in {"request", "req"} and isinstance(f_locals[key], class_type): return f_locals[key] if f.f_back is not None: f = f.f_back @@ -206,4 +214,4 @@ def _get_correlation_id_in_request_header(request_adapter, request): def is_not_match_any_pattern(path, patterns): - return all(map(lambda pattern: re.search(pattern, path) is None, patterns)) \ No newline at end of file + return all(map(lambda pattern: re.search(pattern, path) is None, patterns)) diff --git a/tests-performance/benmark_micro.py b/tests-performance/benmark_micro.py index 4fb6438..c9bb171 100644 --- a/tests-performance/benmark_micro.py +++ b/tests-performance/benmark_micro.py @@ -35,12 +35,14 @@ print(timeit1) print((utcnow - _epoch).total_seconds()) -timeit2 = timeit.timeit(lambda: int((utcnow - _epoch).total_seconds()) * 1000000000 + utcnow.microsecond * 1000, - number=numbers) +timeit2 = timeit.timeit( + lambda: int((utcnow - _epoch).total_seconds()) * 1000000000 + utcnow.microsecond * 1000, number=numbers +) print(timeit2) -timeit3 = timeit.timeit(lambda: (int((utcnow - _epoch).total_seconds()) * 1000000 + utcnow.microsecond) * 1000, - number=numbers) +timeit3 = timeit.timeit( + lambda: (int((utcnow - _epoch).total_seconds()) * 1000000 + utcnow.microsecond) * 1000, number=numbers +) print(timeit3) # 1456820553816849408 # 1496635859 diff --git a/tests-performance/flask_performance_test_pure_python.py b/tests-performance/flask_performance_test_pure_python.py index dd39e71..1197c23 100644 --- a/tests-performance/flask_performance_test_pure_python.py +++ b/tests-performance/flask_performance_test_pure_python.py @@ -13,24 +13,31 @@ logger.addHandler(StreamHandler(sys.stdout)) app = Flask(__name__) -FORMAT = 'important: %s' -MESSAGE = 'some important information to be logged' +FORMAT = "important: %s" +MESSAGE = "some important information to be logged" -@app.route('/') +@app.route("/") def home________________(): count_ = 10 - if 'count' in request.args: - count_ = int(request.args['count']) + if "count" in request.args: + count_ = int(request.args["count"]) simple_logging = Timer(lambda: logger.info(MESSAGE)).timeit(number=count_) formatted_logging = Timer(lambda: logger.info(FORMAT, MESSAGE)).timeit(number=count_) - return "simple_logging : " + str(simple_logging) + "
" \ - + "formatted_logging " + str(formatted_logging) + "
" \ - + "aggregated " + str(formatted_logging) + return ( + "simple_logging : " + + str(simple_logging) + + "
" + + "formatted_logging " + + str(formatted_logging) + + "
" + + "aggregated " + + str(formatted_logging) + ) -port = os.getenv('PORT', '5000') +port = os.getenv("PORT", "5000") if __name__ == "__main__": - app.debug = not os.getenv('PORT') + app.debug = not os.getenv("PORT") logger.info("App started") - app.run(host='0.0.0.0', port=int(port), use_reloader=False) + app.run(host="0.0.0.0", port=int(port), use_reloader=False) diff --git a/tests-performance/flask_performance_test_v2.py b/tests-performance/flask_performance_test_v2.py index 04a32ee..3ed5497 100644 --- a/tests-performance/flask_performance_test_v2.py +++ b/tests-performance/flask_performance_test_v2.py @@ -17,24 +17,31 @@ json_logging.init_flask() json_logging.init_request_instrument(app) -FORMAT = 'important: %s' -MESSAGE = 'some important information to be logged' +FORMAT = "important: %s" +MESSAGE = "some important information to be logged" -@app.route('/') +@app.route("/") def home________________(): count_ = 10 - if 'count' in request.args: - count_ = int(request.args['count']) + if "count" in request.args: + count_ = int(request.args["count"]) simple_logging = Timer(lambda: logger.info(MESSAGE)).timeit(number=count_) formatted_logging = Timer(lambda: logger.info(FORMAT, MESSAGE)).timeit(number=count_) - return "simple_logging : " + str(simple_logging) + "
" \ - + "formatted_logging " + str(formatted_logging) + "
" \ - + "aggregated " + str(formatted_logging) - - -port = os.getenv('PORT', '5002') + return ( + "simple_logging : " + + str(simple_logging) + + "
" + + "formatted_logging " + + str(formatted_logging) + + "
" + + "aggregated " + + str(formatted_logging) + ) + + +port = os.getenv("PORT", "5002") if __name__ == "__main__": - app.debug = not os.getenv('PORT') + app.debug = not os.getenv("PORT") logger.info("App started") - app.run(host='0.0.0.0', port=int(port), use_reloader=False) + app.run(host="0.0.0.0", port=int(port), use_reloader=False) diff --git a/tests/conftest.py b/tests/conftest.py index a3e80b9..3753bcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Global fixtures and settings for the pytest test suite""" -import sys + import os +import sys # Add test helper modules to search path with out making "tests" a Python package sys.path.append(os.path.join(os.path.dirname(__file__), "helpers")) diff --git a/tests/helpers/imports.py b/tests/helpers/imports.py index a1d6234..6e19569 100644 --- a/tests/helpers/imports.py +++ b/tests/helpers/imports.py @@ -1,9 +1,10 @@ """Helper functions related to module imports""" + import sys def undo_imports_from_package(package: str): """Removes all imported modules from the given package from sys.modules""" for k in sorted(sys.modules.keys(), key=lambda s: len(s), reverse=True): - if k == package or k.startswith(package + '.'): + if k == package or k.startswith(package + "."): del sys.modules[k] diff --git a/tests/smoketests/fastapi/api.py b/tests/smoketests/fastapi/api.py index 7d57be2..d33c71c 100644 --- a/tests/smoketests/fastapi/api.py +++ b/tests/smoketests/fastapi/api.py @@ -1,5 +1,12 @@ +import datetime +import logging import os -import datetime, logging, sys, json_logging, fastapi, uvicorn +import sys + +import fastapi +import uvicorn + +import json_logging app = fastapi.FastAPI() json_logging.init_fastapi(enable_json=True) @@ -10,13 +17,15 @@ logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.get('/') + +@app.get("/") def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() return "Hello world : " + str(datetime.datetime.now()) + if __name__ == "__main__": - port = os.getenv('PORT', '5000') - uvicorn.run(app, host='0.0.0.0', port=int(port)) + port = os.getenv("PORT", "5000") + uvicorn.run(app, host="0.0.0.0", port=int(port)) diff --git a/tests/smoketests/flask/api.py b/tests/smoketests/flask/api.py index d4bfb0a..d2d7bcb 100644 --- a/tests/smoketests/flask/api.py +++ b/tests/smoketests/flask/api.py @@ -1,5 +1,11 @@ +import datetime +import logging import os -import datetime, logging, sys, json_logging, flask +import sys + +import flask + +import json_logging app = flask.Flask(__name__) json_logging.init_flask(enable_json=True) @@ -10,13 +16,15 @@ logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.route('/') + +@app.route("/") def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() return "Hello world : " + str(datetime.datetime.now()) + if __name__ == "__main__": - port = os.getenv('PORT', '5000') - app.run(host='0.0.0.0', port=int(port), use_reloader=False) + port = os.getenv("PORT", "5000") + app.run(host="0.0.0.0", port=int(port), use_reloader=False) diff --git a/tests/smoketests/quart/api.py b/tests/smoketests/quart/api.py index 5cf09ef..5599914 100644 --- a/tests/smoketests/quart/api.py +++ b/tests/smoketests/quart/api.py @@ -1,5 +1,11 @@ +import asyncio +import logging import os -import asyncio, logging, sys, json_logging, quart +import sys + +import quart + +import json_logging app = quart.Quart(__name__) json_logging.init_quart(enable_json=True) @@ -10,14 +16,16 @@ logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) -@app.route('/') + +@app.route("/") async def home(): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) correlation_id = json_logging.get_correlation_id() return "Hello world" + if __name__ == "__main__": loop = asyncio.get_event_loop() - port = os.getenv('PORT', '5000') - app.run(host='0.0.0.0', port=int(port), use_reloader=False, loop=loop) + port = os.getenv("PORT", "5000") + app.run(host="0.0.0.0", port=int(port), use_reloader=False, loop=loop) diff --git a/tests/smoketests/sanic/api.py b/tests/smoketests/sanic/api.py index d543f21..020b457 100644 --- a/tests/smoketests/sanic/api.py +++ b/tests/smoketests/sanic/api.py @@ -1,5 +1,10 @@ +import logging import os -import logging, sys, json_logging, sanic +import sys + +import sanic + +import json_logging app = sanic.Sanic(name="sanic-web-app") json_logging.init_sanic(enable_json=True) @@ -10,10 +15,11 @@ logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) + @app.route("/") async def home(request): logger.info("test log statement") - logger.info("test log statement with extra props", extra={'props': {"extra_property": 'extra_value'}}) + logger.info("test log statement with extra props", extra={"props": {"extra_property": "extra_value"}}) # this will be faster correlation_id = json_logging.get_correlation_id(request=request) # this will be slower, but will work in context you cant get a reference of request object @@ -21,6 +27,7 @@ async def home(request): return sanic.response.text("hello world") + if __name__ == "__main__": - port = os.getenv('PORT', '5000') + port = os.getenv("PORT", "5000") app.run(host="0.0.0.0", port=int(port)) diff --git a/tests/smoketests/test_run_smoketest.py b/tests/smoketests/test_run_smoketest.py index 5e3842d..be97153 100644 --- a/tests/smoketests/test_run_smoketest.py +++ b/tests/smoketests/test_run_smoketest.py @@ -21,16 +21,16 @@ def collect_backends(): If the environment variable "backend" is set, only this one backend is returned """ - preset_backend = os.environ.get('backend') + preset_backend = os.environ.get("backend") if preset_backend: yield preset_backend else: for folder in Path(__file__).parent.iterdir(): - if folder.is_dir() and folder.name != '__pycache__': + if folder.is_dir() and folder.name != "__pycache__": yield str(folder.name) -@pytest.mark.parametrize('backend', collect_backends(), ids=collect_backends()) +@pytest.mark.parametrize("backend", collect_backends(), ids=collect_backends()) def test_api_example(backend): """For each of the examples start the API and see if the root endpoint is reachable""" api_process = subprocess.Popen( @@ -44,10 +44,10 @@ def test_api_example(backend): time.sleep(3) session = requests.Session() session.trust_env = False - os.environ['NO_PROXY'] = 'localhost' - os.environ['no_proxy'] = 'localhost' + os.environ["NO_PROXY"] = "localhost" + os.environ["no_proxy"] = "localhost" try: - port = os.getenv('PORT', '5000') + port = os.getenv("PORT", "5000") response = requests.get(f"http://localhost:{port}/", timeout=1) assert response.status_code == 200 return diff --git a/tests/test_fastapi.py b/tests/test_fastapi.py index 73b5750..8b02baf 100644 --- a/tests/test_fastapi.py +++ b/tests/test_fastapi.py @@ -1,4 +1,5 @@ """Test suite for the flask backend""" + import json import logging import pathlib @@ -73,9 +74,9 @@ async def log_exception(): @app.get("/get-correlation-id") async def get_correlation_id(): - return {'correlation_id': json_logging.get_correlation_id()} + return {"correlation_id": json_logging.get_correlation_id()} - @app.get('/no-request-instrumentation') + @app.get("/no-request-instrumentation") async def excluded_from_request_instrumentation(): return {} diff --git a/tests/test_flask.py b/tests/test_flask.py index ecf0186..bdc5c4b 100644 --- a/tests/test_flask.py +++ b/tests/test_flask.py @@ -1,4 +1,5 @@ """Test suite for the flask backend""" + import json import logging import pathlib @@ -6,7 +7,6 @@ import flask import pytest - from helpers import constants from helpers.handler import FormattedMessageCollectorHandler from helpers.imports import undo_imports_from_package @@ -73,9 +73,9 @@ def log_exception(): @app.route("/get-correlation-id") def get_correlation_id(): - return {'correlation_id': json_logging.get_correlation_id()} + return {"correlation_id": json_logging.get_correlation_id()} - @app.route('/no-request-instrumentation') + @app.route("/no-request-instrumentation") def excluded_from_request_instrumentation(): return {} From 19a0bc17d1523afd482ee8a9d477e5560bd6fd91 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:50:09 -0700 Subject: [PATCH 6/8] upgrade github workflow config code_quality.yml --- .github/workflows/code_quality.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 64b1dcf..490ccb8 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -13,9 +13,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install dependencies @@ -36,15 +36,15 @@ jobs: strategy: matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: "Git checkout" - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install dependencies run: | python -m pip install --upgrade pip @@ -63,11 +63,11 @@ jobs: steps: - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: "Git checkout" - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Install dependencies run: | python -m pip install --upgrade pip From cd29cb995d3f04362ae05896ae63dfd6a86c6d74 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 15:54:43 -0700 Subject: [PATCH 7/8] switching to main branch --- .github/workflows/code_quality.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 490ccb8..85867c7 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -2,9 +2,10 @@ name: Code Quality on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] + workflow_dispatch: jobs: flake8: From f9f30b2ca09df0e380409ad5797aa1e11a10abb8 Mon Sep 17 00:00:00 2001 From: Viraj Kanwade Date: Mon, 9 Sep 2024 16:29:49 -0700 Subject: [PATCH 8/8] Support Python 3.9 and above --- tests/helpers/constants.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index 3b6cfab..33f8422 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -1,6 +1,8 @@ """Constants shared by multiple tests""" -STANDARD_MSG_ATTRIBUTES = { +import sys + +_msg_attrs = [ "written_at", "written_ts", "msg", @@ -11,5 +13,9 @@ "module", "line_no", "correlation_id", - "taskName", -} +] + +if sys.version_info.major == 3 and sys.version_info.minor >= 12: + _msg_attrs.append("taskName") + +STANDARD_MSG_ATTRIBUTES = set(_msg_attrs)