From 60893a5cb0e01dd9c4f5300cd940796687ebb329 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 22:35:51 +1000 Subject: [PATCH 01/27] Refactor into core and json module, add orjson support --- pylintrc | 2 +- pyproject.toml | 16 +- src/pythonjsonlogger/core.py | 318 +++++++++++++++++++++++++++++ src/pythonjsonlogger/json.py | 102 +++++++++ src/pythonjsonlogger/jsonlogger.py | 304 --------------------------- src/pythonjsonlogger/orjson.py | 47 +++++ tests/test_jsonlogger.py | 48 ++--- tox.ini | 6 +- 8 files changed, 505 insertions(+), 338 deletions(-) create mode 100644 src/pythonjsonlogger/core.py create mode 100644 src/pythonjsonlogger/json.py delete mode 100644 src/pythonjsonlogger/jsonlogger.py create mode 100644 src/pythonjsonlogger/orjson.py diff --git a/pylintrc b/pylintrc index c2f821e..79541d9 100644 --- a/pylintrc +++ b/pylintrc @@ -3,7 +3,7 @@ # A comma-separated list of package or module names from where C extensions may # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. -extension-pkg-whitelist= +extension-pkg-whitelist=orjson # Add files or directories to the blacklist. They should be base names, not # paths. diff --git a/pyproject.toml b/pyproject.toml index 63266ee..bcb91ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,11 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.0.1" +version = "4.0.0.dev1" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, + {name = "Nicholas Hairs", email = "info+python-json-logger@nicholashairs.com"}, ] maintainers = [ {name = "Nicholas Hairs", email = "info+python-json-logger@nicholashairs.com"}, @@ -15,7 +16,9 @@ maintainers = [ # Dependency Information requires-python = ">=3.7" -# dependencies = [] +dependencies = [ + "typing_extensions", +] # Extra information readme = "README.md" @@ -41,14 +44,15 @@ classifiers = [ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] -lint = [ +dev = [ + ## Optional but required for dev + "orjson", + ## Lint "validate-pyproject[all]", "black", "pylint", "mypy", -] - -test = [ + ## Test "pytest", ] diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py new file mode 100644 index 0000000..19ec4c8 --- /dev/null +++ b/src/pythonjsonlogger/core.py @@ -0,0 +1,318 @@ +"""Core functionality shared by all JSON loggers""" + +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +from datetime import datetime, timezone +import importlib +import logging +import re +import sys +from typing import Optional, Union, Callable, List, Dict, Container, Any, Sequence + +if sys.version_info >= (3, 10): + from typing import TypeAlias +else: + from typing_extensions import TypeAlias + +## Installed + +## Application + + +### CONSTANTS +### ============================================================================ +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +# Changed in 3.0.0, is now list[str] instead of tuple[str, ...] +RESERVED_ATTRS: List[str] = [ + "args", + "asctime", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", +] + +if sys.version_info >= (3, 12): + # taskName added in python 3.12 + RESERVED_ATTRS.append("taskName") + RESERVED_ATTRS.sort() + + +STYLE_STRING_TEMPLATE_REGEX = re.compile(r"\$\{(.+?)\}", re.IGNORECASE) +STYLE_STRING_FORMAT_REGEX = re.compile(r"\{(.+?)\}", re.IGNORECASE) +STYLE_PERCENT_REGEX = re.compile(r"%\((.+?)\)", re.IGNORECASE) + +## Type Aliases +## ----------------------------------------------------------------------------- +OptionalCallableOrStr: TypeAlias = Optional[Union[Callable, str]] +LogRecord: TypeAlias = Dict[str, Any] + + +### FUNCTIONS +### ============================================================================ +def str_to_object(obj: Any) -> Any: + """Import strings to an object, leaving non-strings as-is. + + Args: + obj: the object or string to process + + *New in 4.0* + """ + + if not isinstance(obj, str): + return obj + + module_name, attribute_name = obj.rsplit(".", 1) + return getattr(importlib.import_module(module_name), attribute_name) + + +def merge_record_extra( + record: logging.LogRecord, + target: Dict, + reserved: Container[str], + rename_fields: Optional[Dict[str, str]] = None, +) -> Dict: + """ + Merges extra attributes from LogRecord object into target dictionary + + :param record: logging.LogRecord + :param target: dict to update + :param reserved: dict or list with reserved keys to skip + :param rename_fields: an optional dict, used to rename field names in the output. + Rename levelname to log.level: {'levelname': 'log.level'} + + *Changed in 4.0*: `reserved` is now `Container[str]`. + """ + if rename_fields is None: + rename_fields = {} + for key, value in record.__dict__.items(): + # this allows to have numeric keys + if key not in reserved and not (hasattr(key, "startswith") and key.startswith("_")): + target[rename_fields.get(key, key)] = value + return target + + +### CLASSES +### ============================================================================ +class BaseJsonFormatter(logging.Formatter): + """Base class for pythonjsonlogger formatters + + Must not be used directly + """ + + _style: Union[logging.PercentStyle, str] # type: ignore[assignment] + + ## Parent Methods + ## ------------------------------------------------------------------------- + # pylint: disable=too-many-arguments,super-init-not-called + def __init__( + self, + fmt: Optional[str] = None, + datefmt: Optional[str] = None, + style: str = "%", + validate: bool = True, + *, + defaults: None = None, + prefix: str = "", + rename_fields: Optional[Dict[str, str]] = None, + static_fields: Optional[Dict[str, Any]] = None, + reserved_attrs: Optional[Sequence[str]] = None, + timestamp: Union[bool, str] = False, + ) -> None: + """ + Args: + fmt: string representing fields to log + datefmt: format to use when formatting asctime field + style: how to extract log fields from `fmt` + validate: validate fmt against style, if implementing a custom style you + must set this to `False`. + defaults: ignored - kept for compatibility + prefix: an optional string prefix added at the beginning of + the formatted string + rename_fields: an optional dict, used to rename field names in the output. + Rename message to @message: {'message': '@message'} + static_fields: an optional dict, used to add fields with static values to all logs + reserved_attrs: an optional list of fields that will be skipped when + outputting json log record. Defaults to all log record attributes: + http://docs.python.org/library/logging.html#logrecord-attributes + timestamp: an optional string/boolean field to add a timestamp when + outputting the json log record. If string is passed, timestamp will be added + to log record using string as key. If True boolean is passed, timestamp key + will be "timestamp". Defaults to False/off. + + *Changed in 4.0*: you can now use custom values for style by setting validate to `False`. + The value is stored in `self._style` as a string. The `parse` method will need to be + overridden in order to support the new style. + """ + ## logging.Formatter compatibility + ## --------------------------------------------------------------------- + if style in logging._STYLES: + _style = logging._STYLES[style][0](fmt, defaults=defaults) # type: ignore[operator] + if validate: + _style.validate() + self._style = _style + self._fmt = _style._fmt + + elif not validate: + self._style = style + self._fmt = fmt + + else: + raise ValueError(f"Style must be one of: {','.join(logging._STYLES.keys())}") + + self.datefmt = datefmt + + ## JSON Logging specific + ## --------------------------------------------------------------------- + self.prefix = prefix + self.rename_fields = rename_fields if rename_fields is not None else {} + self.static_fields = static_fields if static_fields is not None else {} + self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS) + self.timestamp = timestamp + + self._required_fields = self.parse() + self._skip_fields = set(self._required_fields) + self._skip_fields.update(self.reserved_attrs) + return + + def format(self, record: logging.LogRecord) -> str: + """Formats a log record and serializes to json""" + message_dict: Dict[str, Any] = {} + # TODO: logging.LogRecord.msg and logging.LogRecord.message in typeshed + # are always type of str. We shouldn't need to override that. + if isinstance(record.msg, dict): + message_dict = record.msg + record.message = "" + else: + record.message = record.getMessage() + # only format time if needed + if "asctime" in self._required_fields: + record.asctime = self.formatTime(record, self.datefmt) + + # Display formatted exception, but allow overriding it in the + # user-supplied dict. + if record.exc_info and not message_dict.get("exc_info"): + message_dict["exc_info"] = self.formatException(record.exc_info) + if not message_dict.get("exc_info") and record.exc_text: + message_dict["exc_info"] = record.exc_text + # Display formatted record of stack frames + # default format is a string returned from :func:`traceback.print_stack` + if record.stack_info and not message_dict.get("stack_info"): + message_dict["stack_info"] = self.formatStack(record.stack_info) + + log_record: LogRecord = {} + self.add_fields(log_record, record, message_dict) + log_record = self.process_log_record(log_record) + + return self.serialize_log_record(log_record) + + ## JSON Formatter Specific Methods + ## ------------------------------------------------------------------------- + def parse(self) -> List[str]: + """Parses format string looking for substitutions + + This method is responsible for returning a list of fields (as strings) + to include in all log messages. + + You can support custom styles by overriding this method. + """ + if isinstance(self._style, logging.StringTemplateStyle): + formatter_style_pattern = STYLE_STRING_TEMPLATE_REGEX + + elif isinstance(self._style, logging.StrFormatStyle): + formatter_style_pattern = STYLE_STRING_FORMAT_REGEX + + elif isinstance(self._style, logging.PercentStyle): + # PercentStyle is parent class of StringTemplateStyle and StrFormatStyle + # so it must be checked last. + formatter_style_pattern = STYLE_PERCENT_REGEX + + else: + raise ValueError(f"Style {self._style!r} is not supported") + + if self._fmt: + return formatter_style_pattern.findall(self._fmt) + + return [] + + def serialize_log_record(self, log_record: LogRecord) -> str: + """Returns the final representation of the log record. + + Args: + log_record: the log record + """ + return self.prefix + self.jsonify_log_record(log_record) + + def add_fields( + self, + log_record: Dict[str, Any], + record: logging.LogRecord, + message_dict: Dict[str, Any], + ) -> None: + """ + Override this method to implement custom logic for adding fields. + """ + for field in self._required_fields: + log_record[field] = record.__dict__.get(field) + + log_record.update(self.static_fields) + log_record.update(message_dict) + merge_record_extra( + record, + log_record, + reserved=self._skip_fields, + rename_fields=self.rename_fields, + ) + + if self.timestamp: + # TODO: Can this use isinstance instead? + # pylint: disable=unidiomatic-typecheck + key = self.timestamp if type(self.timestamp) == str else "timestamp" + log_record[key] = datetime.fromtimestamp(record.created, tz=timezone.utc) + + self._perform_rename_log_fields(log_record) + return + + def _perform_rename_log_fields(self, log_record: Dict[str, Any]) -> None: + for old_field_name, new_field_name in self.rename_fields.items(): + log_record[new_field_name] = log_record[old_field_name] + del log_record[old_field_name] + return + + # Child Methods + # .......................................................................... + def jsonify_log_record(self, log_record: LogRecord) -> str: + """Convert this log record into a JSON string. + + Child classes MUST override this method. + """ + raise NotImplementedError() + + def process_log_record(self, log_record: LogRecord) -> LogRecord: + """Custom processing of the log record. + + Child classes can override this method to alter the log record before it + is serialized. + """ + return log_record diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py new file mode 100644 index 0000000..e248347 --- /dev/null +++ b/src/pythonjsonlogger/json.py @@ -0,0 +1,102 @@ +"""JSON Formatter using the standard library `json` module""" + +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +from datetime import date, datetime, time +from inspect import istraceback +import json +import traceback +from typing import Any, Callable, Optional, Union + +## Application +from . import core + + +### CLASSES +### ============================================================================ +class JsonEncoder(json.JSONEncoder): + """ + A custom encoder extending the default JSONEncoder + """ + + def default(self, o: Any) -> Any: + if isinstance(o, (date, datetime, time)): + return self.format_datetime_obj(o) + + if istraceback(o): + return "".join(traceback.format_tb(o)).strip() + + # pylint: disable=unidiomatic-typecheck + if type(o) == Exception or isinstance(o, Exception) or type(o) == type: + return str(o) + + try: + return super().default(o) + + except TypeError: + try: + return str(o) + + except Exception: # pylint: disable=broad-exception-caught + return None + + def format_datetime_obj(self, o): + """Format datetime objects found in self.default + + This allows subclasses to change the datetime format without understanding the + internals of the default method. + """ + return o.isoformat() + + +class JsonFormatter(core.BaseJsonFormatter): + """ + A custom formatter to format logging records as json strings. + Extra values will be formatted as str() if not supported by + json default encoder + """ + + # pylint: disable=too-many-arguments + def __init__( + self, + *args, + json_default: core.OptionalCallableOrStr = None, + json_encoder: core.OptionalCallableOrStr = None, + json_serializer: Union[Callable, str] = json.dumps, + json_indent: Optional[Union[int, str]] = None, + json_ensure_ascii: bool = True, + **kwargs, + ) -> None: + """ + :param json_default: a function for encoding non-standard objects + as outlined in https://docs.python.org/3/library/json.html + :param json_encoder: optional custom encoder + :param json_serializer: a :meth:`json.dumps`-compatible callable + that will be used to serialize the log record. + :param json_indent: indent parameter for json.dumps + :param json_ensure_ascii: ensure_ascii parameter for json.dumps + """ + super().__init__(*args, **kwargs) + + self.json_default = core.str_to_object(json_default) + self.json_encoder = core.str_to_object(json_encoder) + self.json_serializer = core.str_to_object(json_serializer) + self.json_indent = json_indent + self.json_ensure_ascii = json_ensure_ascii + if not self.json_encoder and not self.json_default: + self.json_encoder = JsonEncoder + return + + def jsonify_log_record(self, log_record: core.LogRecord) -> str: + """Returns a json string of the log record.""" + return self.json_serializer( + log_record, + default=self.json_default, + cls=self.json_encoder, + indent=self.json_indent, + ensure_ascii=self.json_ensure_ascii, + ) diff --git a/src/pythonjsonlogger/jsonlogger.py b/src/pythonjsonlogger/jsonlogger.py deleted file mode 100644 index 349564a..0000000 --- a/src/pythonjsonlogger/jsonlogger.py +++ /dev/null @@ -1,304 +0,0 @@ -""" -This library is provided to allow standard python logging -to output log data as JSON formatted strings -""" - -import logging -import json -import re -import traceback -import importlib -from datetime import date, datetime, time, timezone -import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Union - -from inspect import istraceback - -from collections import OrderedDict - -# skip natural LogRecord attributes -# http://docs.python.org/library/logging.html#logrecord-attributes -# Changed in 3.0.0, is now list[str] instead of tuple[str, ...] -RESERVED_ATTRS: List[str] = [ - "args", - "asctime", - "created", - "exc_info", - "exc_text", - "filename", - "funcName", - "levelname", - "levelno", - "lineno", - "module", - "msecs", - "message", - "msg", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "stack_info", - "thread", - "threadName", -] - -if sys.version_info >= (3, 12): - # taskName added in python 3.12 - RESERVED_ATTRS.append("taskName") - RESERVED_ATTRS.sort() - -OptionalCallableOrStr = Optional[Union[Callable, str]] - - -def merge_record_extra( - record: logging.LogRecord, - target: Dict, - reserved: Union[Dict, List], - rename_fields: Optional[Dict[str, str]] = None, -) -> Dict: - """ - Merges extra attributes from LogRecord object into target dictionary - - :param record: logging.LogRecord - :param target: dict to update - :param reserved: dict or list with reserved keys to skip - :param rename_fields: an optional dict, used to rename field names in the output. - Rename levelname to log.level: {'levelname': 'log.level'} - """ - if rename_fields is None: - rename_fields = {} - for key, value in record.__dict__.items(): - # this allows to have numeric keys - if key not in reserved and not (hasattr(key, "startswith") and key.startswith("_")): - target[rename_fields.get(key, key)] = value - return target - - -class JsonEncoder(json.JSONEncoder): - """ - A custom encoder extending the default JSONEncoder - """ - - def default(self, o: Any) -> Any: - if isinstance(o, (date, datetime, time)): - return self.format_datetime_obj(o) - - if istraceback(o): - return "".join(traceback.format_tb(o)).strip() - - # pylint: disable=unidiomatic-typecheck - if type(o) == Exception or isinstance(o, Exception) or type(o) == type: - return str(o) - - try: - return super().default(o) - - except TypeError: - try: - return str(o) - - except Exception: # pylint: disable=broad-exception-caught - return None - - def format_datetime_obj(self, o): - """Format datetime objects found in self.default - - This allows subclasses to change the datetime format without understanding the - internals of the default method. - """ - return o.isoformat() - - -class JsonFormatter(logging.Formatter): - """ - A custom formatter to format logging records as json strings. - Extra values will be formatted as str() if not supported by - json default encoder - """ - - # pylint: disable=too-many-arguments - def __init__( - self, - *args: Any, - json_default: OptionalCallableOrStr = None, - json_encoder: OptionalCallableOrStr = None, - json_serializer: Union[Callable, str] = json.dumps, - json_indent: Optional[Union[int, str]] = None, - json_ensure_ascii: bool = True, - prefix: str = "", - rename_fields: Optional[dict] = None, - static_fields: Optional[dict] = None, - reserved_attrs: Union[Tuple[str, ...], List[str], None] = None, - timestamp: Union[bool, str] = False, - **kwargs: Any, - ) -> None: - """ - :param json_default: a function for encoding non-standard objects - as outlined in https://docs.python.org/3/library/json.html - :param json_encoder: optional custom encoder - :param json_serializer: a :meth:`json.dumps`-compatible callable - that will be used to serialize the log record. - :param json_indent: indent parameter for json.dumps - :param json_ensure_ascii: ensure_ascii parameter for json.dumps - :param prefix: an optional string prefix added at the beginning of - the formatted string - :param rename_fields: an optional dict, used to rename field names in the output. - Rename message to @message: {'message': '@message'} - :param static_fields: an optional dict, used to add fields with static values to all logs - :param reserved_attrs: an optional list of fields that will be skipped when - outputting json log record. Defaults to all log record attributes: - http://docs.python.org/library/logging.html#logrecord-attributes - :param timestamp: an optional string/boolean field to add a timestamp when - outputting the json log record. If string is passed, timestamp will be added - to log record using string as key. If True boolean is passed, timestamp key - will be "timestamp". Defaults to False/off. - """ - self.json_default = self._str_to_fn(json_default) - self.json_encoder = self._str_to_fn(json_encoder) - self.json_serializer = self._str_to_fn(json_serializer) - self.json_indent = json_indent - self.json_ensure_ascii = json_ensure_ascii - self.prefix = prefix - self.rename_fields = rename_fields or {} - self.static_fields = static_fields or {} - if reserved_attrs is None: - reserved_attrs = RESERVED_ATTRS - self.reserved_attrs = dict(zip(reserved_attrs, reserved_attrs)) - self.timestamp = timestamp - - # super(JsonFormatter, self).__init__(*args, **kwargs) - logging.Formatter.__init__(self, *args, **kwargs) - if not self.json_encoder and not self.json_default: - self.json_encoder = JsonEncoder - - self._required_fields = self.parse() - self._skip_fields = dict(zip(self._required_fields, self._required_fields)) - self._skip_fields.update(self.reserved_attrs) - return - - def _str_to_fn(self, fn_as_str): - """ - If the argument is not a string, return whatever was passed in. - Parses a string such as package.module.function, imports the module - and returns the function. - - :param fn_as_str: The string to parse. If not a string, return it. - """ - if not isinstance(fn_as_str, str): - return fn_as_str - - path, _, function = fn_as_str.rpartition(".") - module = importlib.import_module(path) - return getattr(module, function) - - def parse(self) -> List[str]: - """ - Parses format string looking for substitutions - - This method is responsible for returning a list of fields (as strings) - to include in all log messages. - """ - if isinstance(self._style, logging.StringTemplateStyle): - formatter_style_pattern = re.compile(r"\$\{(.+?)\}", re.IGNORECASE) - elif isinstance(self._style, logging.StrFormatStyle): - formatter_style_pattern = re.compile(r"\{(.+?)\}", re.IGNORECASE) - # PercentStyle is parent class of StringTemplateStyle and StrFormatStyle so - # it needs to be checked last. - elif isinstance(self._style, logging.PercentStyle): - formatter_style_pattern = re.compile(r"%\((.+?)\)", re.IGNORECASE) - else: - raise ValueError(f"Invalid format: {self._fmt!r}") - - if self._fmt: - return formatter_style_pattern.findall(self._fmt) - return [] - - def add_fields( - self, - log_record: Dict[str, Any], - record: logging.LogRecord, - message_dict: Dict[str, Any], - ) -> None: - """ - Override this method to implement custom logic for adding fields. - """ - for field in self._required_fields: - log_record[field] = record.__dict__.get(field) - - log_record.update(self.static_fields) - log_record.update(message_dict) - merge_record_extra( - record, - log_record, - reserved=self._skip_fields, - rename_fields=self.rename_fields, - ) - - if self.timestamp: - # TODO: Can this use isinstance instead? - # pylint: disable=unidiomatic-typecheck - key = self.timestamp if type(self.timestamp) == str else "timestamp" - log_record[key] = datetime.fromtimestamp(record.created, tz=timezone.utc) - - self._perform_rename_log_fields(log_record) - return - - def _perform_rename_log_fields(self, log_record: Dict[str, Any]) -> None: - for old_field_name, new_field_name in self.rename_fields.items(): - log_record[new_field_name] = log_record[old_field_name] - del log_record[old_field_name] - return - - def process_log_record(self, log_record: Dict[str, Any]) -> Dict[str, Any]: - """ - Override this method to implement custom logic - on the possibly ordered dictionary. - """ - return log_record - - def jsonify_log_record(self, log_record: Dict[str, Any]) -> str: - """Returns a json string of the log record.""" - return self.json_serializer( - log_record, - default=self.json_default, - cls=self.json_encoder, - indent=self.json_indent, - ensure_ascii=self.json_ensure_ascii, - ) - - def serialize_log_record(self, log_record: Dict[str, Any]) -> str: - """Returns the final representation of the log record.""" - return self.prefix + self.jsonify_log_record(log_record) - - def format(self, record: logging.LogRecord) -> str: - """Formats a log record and serializes to json""" - message_dict: Dict[str, Any] = {} - # TODO: logging.LogRecord.msg and logging.LogRecord.message in typeshed - # are always type of str. We shouldn't need to override that. - if isinstance(record.msg, dict): - message_dict = record.msg - record.message = "" - else: - record.message = record.getMessage() - # only format time if needed - if "asctime" in self._required_fields: - record.asctime = self.formatTime(record, self.datefmt) - - # Display formatted exception, but allow overriding it in the - # user-supplied dict. - if record.exc_info and not message_dict.get("exc_info"): - message_dict["exc_info"] = self.formatException(record.exc_info) - if not message_dict.get("exc_info") and record.exc_text: - message_dict["exc_info"] = record.exc_text - # Display formatted record of stack frames - # default format is a string returned from :func:`traceback.print_stack` - if record.stack_info and not message_dict.get("stack_info"): - message_dict["stack_info"] = self.formatStack(record.stack_info) - - log_record: Dict[str, Any] = OrderedDict() - self.add_fields(log_record, record, message_dict) - log_record = self.process_log_record(log_record) - - return self.serialize_log_record(log_record) diff --git a/src/pythonjsonlogger/orjson.py b/src/pythonjsonlogger/orjson.py new file mode 100644 index 0000000..5894c4e --- /dev/null +++ b/src/pythonjsonlogger/orjson.py @@ -0,0 +1,47 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library + +## Installed +import orjson + +## Application +from . import core + + +### CLASSES +### ============================================================================ +class OrjsonFormatter(core.BaseJsonFormatter): + """JSON formatter using orjson for encoding.""" + + # pylint: disable=too-many-arguments + def __init__( + self, + *args, + json_default: core.OptionalCallableOrStr = None, + json_indent: bool = False, + **kwargs, + ) -> None: + """ + Args: + json_default: a function for encoding non-standard objects see: + https://github.com/ijl/orjson#default + json_indent: indent output with 2 spaces. see: + https://github.com/ijl/orjson#opt_indent_2 + """ + super().__init__(*args, **kwargs) + + self.json_default = core.str_to_object(json_default) + self.json_indent = json_indent + return + + def jsonify_log_record(self, log_record: core.LogRecord) -> str: + """Returns a json string of the log record.""" + opt = 0 + if self.json_indent: + opt |= orjson.OPT_INDENT_2 + + return orjson.dumps(log_record, default=self.json_default, option=opt).decode("ascii") diff --git a/tests/test_jsonlogger.py b/tests/test_jsonlogger.py index abd04ba..1d7c491 100644 --- a/tests/test_jsonlogger.py +++ b/tests/test_jsonlogger.py @@ -8,8 +8,8 @@ import unittest import unittest.mock -sys.path.append("src/python-json-logger") -from pythonjsonlogger import jsonlogger +from pythonjsonlogger.core import RESERVED_ATTRS, merge_record_extra +from pythonjsonlogger.json import JsonFormatter class TestJsonLogger(unittest.TestCase): @@ -22,7 +22,7 @@ def setUp(self): self.log.addHandler(self.log_handler) def test_default_format(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) msg = "testing logging format" @@ -32,7 +32,7 @@ def test_default_format(self): self.assertEqual(log_json["message"], msg) def test_percentage_format(self): - fr = jsonlogger.JsonFormatter( + fr = JsonFormatter( # All kind of different styles to check the regex "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" ) @@ -46,7 +46,7 @@ def test_percentage_format(self): self.assertEqual(log_json.keys(), {"levelname", "message", "filename", "lineno", "asctime"}) def test_rename_base_field(self): - fr = jsonlogger.JsonFormatter(rename_fields={"message": "@message"}) + fr = JsonFormatter(rename_fields={"message": "@message"}) self.log_handler.setFormatter(fr) msg = "testing logging format" @@ -56,7 +56,7 @@ def test_rename_base_field(self): self.assertEqual(log_json["@message"], msg) def test_rename_nonexistent_field(self): - fr = jsonlogger.JsonFormatter(rename_fields={"nonexistent_key": "new_name"}) + fr = JsonFormatter(rename_fields={"nonexistent_key": "new_name"}) self.log_handler.setFormatter(fr) stderr_watcher = StringIO() @@ -66,7 +66,7 @@ def test_rename_nonexistent_field(self): self.assertTrue("KeyError: 'nonexistent_key'" in stderr_watcher.getvalue()) def test_add_static_fields(self): - fr = jsonlogger.JsonFormatter(static_fields={"log_stream": "kafka"}) + fr = JsonFormatter(static_fields={"log_stream": "kafka"}) self.log_handler.setFormatter(fr) @@ -101,7 +101,7 @@ def test_format_keys(self): log_format = lambda x: [f"%({i:s})s" for i in x] custom_format = " ".join(log_format(supported_keys)) - fr = jsonlogger.JsonFormatter(custom_format) + fr = JsonFormatter(custom_format) self.log_handler.setFormatter(fr) msg = "testing logging format" @@ -114,7 +114,7 @@ def test_format_keys(self): self.assertTrue(True) def test_unknown_format_key(self): - fr = jsonlogger.JsonFormatter("%(unknown_key)s %(message)s") + fr = JsonFormatter("%(unknown_key)s %(message)s") self.log_handler.setFormatter(fr) msg = "testing unknown logging format" @@ -124,7 +124,7 @@ def test_unknown_format_key(self): self.assertTrue(False, "Should succeed") def test_log_adict(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) msg = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} @@ -138,7 +138,7 @@ def test_log_adict(self): self.assertEqual(log_json["message"], "") def test_log_extra(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) extra = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} @@ -151,7 +151,7 @@ def test_log_extra(self): self.assertEqual(log_json["message"], "hello") def test_json_default_encoder(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) msg = { @@ -169,7 +169,7 @@ def test_json_default_encoder(self): @unittest.mock.patch("time.time", return_value=1500000000.0) def test_json_default_encoder_with_timestamp(self, time_mock): - fr = jsonlogger.JsonFormatter(timestamp=True) + fr = JsonFormatter(timestamp=True) self.log_handler.setFormatter(fr) self.log.info("Hello") @@ -182,7 +182,7 @@ def test_json_custom_default(self): def custom(o): return "very custom" - fr = jsonlogger.JsonFormatter(json_default=custom) + fr = JsonFormatter(json_default=custom) self.log_handler.setFormatter(fr) msg = {"adate": datetime.datetime(1999, 12, 31, 23, 59), "normal": "value"} @@ -192,13 +192,13 @@ def custom(o): self.assertEqual(log_json.get("normal"), "value") def test_json_custom_logic_adds_field(self): - class CustomJsonFormatter(jsonlogger.JsonFormatter): + class CustomJsonFormatter(JsonFormatter): def process_log_record(self, log_record): log_record["custom"] = "value" # Old Style "super" since Python 2.6's logging.Formatter is old # style - return jsonlogger.JsonFormatter.process_log_record(self, log_record) + return JsonFormatter.process_log_record(self, log_record) self.log_handler.setFormatter(CustomJsonFormatter()) self.log.info("message") @@ -218,7 +218,7 @@ def get_traceback_from_exception_followed_by_log_call(self) -> str: return str_traceback def test_exc_info(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) expected_value = self.get_traceback_from_exception_followed_by_log_call() @@ -226,7 +226,7 @@ def test_exc_info(self): self.assertEqual(log_json.get("exc_info"), expected_value) def test_exc_info_renamed(self): - fr = jsonlogger.JsonFormatter("%(exc_info)s", rename_fields={"exc_info": "stack_trace"}) + fr = JsonFormatter("%(exc_info)s", rename_fields={"exc_info": "stack_trace"}) self.log_handler.setFormatter(fr) expected_value = self.get_traceback_from_exception_followed_by_log_call() @@ -235,14 +235,14 @@ def test_exc_info_renamed(self): self.assertEqual(log_json.get("exc_info"), None) def test_ensure_ascii_true(self): - fr = jsonlogger.JsonFormatter() + fr = JsonFormatter() self.log_handler.setFormatter(fr) self.log.info("Привет") msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] self.assertEqual(msg, r"\u041f\u0440\u0438\u0432\u0435\u0442") def test_ensure_ascii_false(self): - fr = jsonlogger.JsonFormatter(json_ensure_ascii=False) + fr = JsonFormatter(json_ensure_ascii=False) self.log_handler.setFormatter(fr) self.log.info("Привет") msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] @@ -256,7 +256,7 @@ def encode_complex(z): type_name = z.__class__.__name__ raise TypeError(f"Object of type '{type_name}' is no JSON serializable") - formatter = jsonlogger.JsonFormatter( + formatter = JsonFormatter( json_default=encode_complex, json_encoder=json.JSONEncoder ) self.log_handler.setFormatter(formatter) @@ -284,9 +284,9 @@ def test_rename_reserved_attrs(self): custom_format = " ".join(log_format(reserved_attrs_map.keys())) reserved_attrs = [ - _ for _ in jsonlogger.RESERVED_ATTRS if _ not in list(reserved_attrs_map.keys()) + attr for attr in RESERVED_ATTRS if attr not in list(reserved_attrs_map.keys()) ] - formatter = jsonlogger.JsonFormatter( + formatter = JsonFormatter( custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map ) self.log_handler.setFormatter(formatter) @@ -302,7 +302,7 @@ def test_merge_record_extra(self): record = logging.LogRecord( "name", level=1, pathname="", lineno=1, msg="Some message", args=None, exc_info=None ) - output = jsonlogger.merge_record_extra(record, target=dict(foo="bar"), reserved=[]) + output = merge_record_extra(record, target=dict(foo="bar"), reserved=[]) self.assertIn("foo", output) self.assertIn("msg", output) self.assertEqual(output["foo"], "bar") diff --git a/tox.ini b/tox.ini index 946be58..14b1e7d 100644 --- a/tox.ini +++ b/tox.ini @@ -17,19 +17,19 @@ python = [testenv] description = run unit tests -extras = test +extras = dev commands = pytest tests [testenv:format] description = run formatters -extras = lint +extras = dev commands = black src tests [testenv:lint] description = run linters -extras = lint +extras = dev commands = validate-pyproject pyproject.toml black --check --diff src tests From 7f7ea2a4f3ca134f4de7ac03802beb999e6a436f Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 22:54:48 +1000 Subject: [PATCH 02/27] Format tests --- tests/test_jsonlogger.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_jsonlogger.py b/tests/test_jsonlogger.py index 1d7c491..e6805a0 100644 --- a/tests/test_jsonlogger.py +++ b/tests/test_jsonlogger.py @@ -256,9 +256,7 @@ def encode_complex(z): type_name = z.__class__.__name__ raise TypeError(f"Object of type '{type_name}' is no JSON serializable") - formatter = JsonFormatter( - json_default=encode_complex, json_encoder=json.JSONEncoder - ) + formatter = JsonFormatter(json_default=encode_complex, json_encoder=json.JSONEncoder) self.log_handler.setFormatter(formatter) value = { From e9aff5ba1858c7fcbbe75b3100d1de599c015abd Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:03:22 +1000 Subject: [PATCH 03/27] Allow tests to complete on all platforms --- .github/workflows/test-suite.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index f4b3b41..082b47c 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -32,6 +32,7 @@ jobs: name: "Python Test ${{matrix.python-version}} ${{ matrix.os }}" needs: [lint] runs-on: "${{ matrix.os }}" + fail-fast: false # allow tests to run on all platforms strategy: matrix: python-version: From eaf56e2ec82dfa47c78bb38abf37951a9b16599e Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:05:35 +1000 Subject: [PATCH 04/27] Fix broken GHA spec --- .github/workflows/test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 082b47c..8152b25 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -32,8 +32,8 @@ jobs: name: "Python Test ${{matrix.python-version}} ${{ matrix.os }}" needs: [lint] runs-on: "${{ matrix.os }}" - fail-fast: false # allow tests to run on all platforms strategy: + fail-fast: false # allow tests to run on all platforms matrix: python-version: - "pypy-3.7" From c36d6da649886288fbfd4a7e43673127dc89e7ce Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:13:38 +1000 Subject: [PATCH 05/27] Don't support defaults kwargs (py310+ only) --- src/pythonjsonlogger/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 19ec4c8..5c25671 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -133,7 +133,6 @@ def __init__( style: str = "%", validate: bool = True, *, - defaults: None = None, prefix: str = "", rename_fields: Optional[Dict[str, str]] = None, static_fields: Optional[Dict[str, Any]] = None, @@ -168,7 +167,9 @@ def __init__( ## logging.Formatter compatibility ## --------------------------------------------------------------------- if style in logging._STYLES: - _style = logging._STYLES[style][0](fmt, defaults=defaults) # type: ignore[operator] + # Note style defaults kwarg is only supported from py310+, since we + # don't use it anyway just ignore it + _style = logging._STYLES[style][0](fmt) # type: ignore[operator] if validate: _style.validate() self._style = _style From 4b1967829521bb6a300f4291a722232823052cc5 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:25:41 +1000 Subject: [PATCH 06/27] Drop py37 support, begin testing again py313 --- .github/workflows/test-suite.yml | 3 +-- pyproject.toml | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 8152b25..73d9b96 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -36,16 +36,15 @@ jobs: fail-fast: false # allow tests to run on all platforms matrix: python-version: - - "pypy-3.7" - "pypy-3.8" - "pypy-3.9" - "pypy-3.10" - - "3.7" - "3.8" - "3.9" - "3.10" - "3.11" - "3.12" + - "3.13" os: - ubuntu-latest - windows-latest diff --git a/pyproject.toml b/pyproject.toml index bcb91ed..5744e39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ maintainers = [ ] # Dependency Information -requires-python = ">=3.7" +requires-python = ">=3.8" dependencies = [ "typing_extensions", ] @@ -29,7 +29,6 @@ classifiers = [ "License :: OSI Approved :: BSD License", "Operating System :: OS Independent", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -44,6 +43,10 @@ classifiers = [ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] +orjson = [ + "orjson", +] + dev = [ ## Optional but required for dev "orjson", From 0d8b8a16b1acd5fd240bdebe707cfcefeea40aa4 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:29:53 +1000 Subject: [PATCH 07/27] Fix py313 in GHA --- .github/workflows/test-suite.yml | 2 +- tox.ini | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 73d9b96..de88c91 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -44,7 +44,7 @@ jobs: - "3.10" - "3.11" - "3.12" - - "3.13" + - "3.13-dev" os: - ubuntu-latest - windows-latest diff --git a/tox.ini b/tox.ini index 14b1e7d..938b9d3 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] requires = tox>=3 -envlist = lint, type, pypy{37,38,39,310}, py{37,38,39,310,311,312} +envlist = lint, type, pypy{37,38,39,310}, py{37,38,39,310,311,312,313} [gh-actions] python = @@ -14,6 +14,7 @@ python = 3.10: py310 3.11: py311 3.12: py312 + 3.13: py312 [testenv] description = run unit tests From 3897a9765d493ee3f2a1d2937eeaa27c39f388b8 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:38:01 +1000 Subject: [PATCH 08/27] Don't instlal python on pypy --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5744e39..34574c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,13 +43,9 @@ classifiers = [ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] -orjson = [ - "orjson", -] - dev = [ ## Optional but required for dev - "orjson", + "orjson;implementation_name!='pypy'", ## Lint "validate-pyproject[all]", "black", From 2eb8820746bfc1a9a5bb9dd1c516eecb8c527849 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:41:39 +1000 Subject: [PATCH 09/27] Run py313 in py313... --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 938b9d3..db04dd8 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ python = 3.10: py310 3.11: py311 3.12: py312 - 3.13: py312 + 3.13: py313 [testenv] description = run unit tests From bc2b96e4fd946ad8f0dc97714be947fb32c68cf3 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sat, 27 Apr 2024 23:51:53 +1000 Subject: [PATCH 10/27] Avoid orjson on python 3.13 while its not supported --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34574c3..f0878fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] dev = [ ## Optional but required for dev - "orjson;implementation_name!='pypy'", + "orjson;implementation_name!='pypy' and python_version<'3.13'", ## Lint "validate-pyproject[all]", "black", From 1441721f7eb8ae626b1202c95abcd17e5997bf6a Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 28 Apr 2024 14:16:03 +1000 Subject: [PATCH 11/27] Migrate tests to pytest, test OrjsonFormatter where possible --- pyproject.toml | 5 + src/pythonjsonlogger/__init__.py | 23 ++ src/pythonjsonlogger/core.py | 7 +- src/pythonjsonlogger/orjson.py | 4 +- tests/test_formatters.py | 375 +++++++++++++++++++++++++++++++ tests/test_jsonlogger.py | 283 ----------------------- 6 files changed, 409 insertions(+), 288 deletions(-) create mode 100644 tests/test_formatters.py diff --git a/pyproject.toml b/pyproject.toml index f0878fb..63abcb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,10 @@ classifiers = [ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] +orjson = [ + "orjson;implementation_name!='pypy'", +] + dev = [ ## Optional but required for dev "orjson;implementation_name!='pypy' and python_version<'3.13'", @@ -53,6 +57,7 @@ dev = [ "mypy", ## Test "pytest", + "freezegun", ] [tool.setuptools.packages.find] diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index e69de29..c43dab1 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -0,0 +1,23 @@ +### IMPORTS +### ============================================================================ +## Future + +## Standard Library +import sys + +## Installed + +## Application + +### CONSTANTS +### ============================================================================ +if sys.implementation.name == "pypy": + # Per https://github.com/ijl/orjson (last checked 2024-04-28) + # > orjson does not and will not support PyPy + ORJSON_AVAILABLE = False +else: + try: + import orjson + ORJSON_AVAILABLE = True + except ImportError: + ORJSON_AVAILABLE = False diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 5c25671..0965864 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -138,6 +138,7 @@ def __init__( static_fields: Optional[Dict[str, Any]] = None, reserved_attrs: Optional[Sequence[str]] = None, timestamp: Union[bool, str] = False, + defaults: Optional[Dict[str, Any]] = None, ) -> None: """ Args: @@ -146,7 +147,7 @@ def __init__( style: how to extract log fields from `fmt` validate: validate fmt against style, if implementing a custom style you must set this to `False`. - defaults: ignored - kept for compatibility + defaults: ignored - kept for compatibility with python 3.10+ prefix: an optional string prefix added at the beginning of the formatted string rename_fields: an optional dict, used to rename field names in the output. @@ -166,9 +167,9 @@ def __init__( """ ## logging.Formatter compatibility ## --------------------------------------------------------------------- + # Note: validate added in 3.8 + # Note: defaults added in 3.10 if style in logging._STYLES: - # Note style defaults kwarg is only supported from py310+, since we - # don't use it anyway just ignore it _style = logging._STYLES[style][0](fmt) # type: ignore[operator] if validate: _style.validate() diff --git a/src/pythonjsonlogger/orjson.py b/src/pythonjsonlogger/orjson.py index 5894c4e..b6317c3 100644 --- a/src/pythonjsonlogger/orjson.py +++ b/src/pythonjsonlogger/orjson.py @@ -40,8 +40,8 @@ def __init__( def jsonify_log_record(self, log_record: core.LogRecord) -> str: """Returns a json string of the log record.""" - opt = 0 + opt = orjson.OPT_NON_STR_KEYS if self.json_indent: opt |= orjson.OPT_INDENT_2 - return orjson.dumps(log_record, default=self.json_default, option=opt).decode("ascii") + return orjson.dumps(log_record, default=self.json_default, option=opt).decode("utf8") diff --git a/tests/test_formatters.py b/tests/test_formatters.py new file mode 100644 index 0000000..a9d06a1 --- /dev/null +++ b/tests/test_formatters.py @@ -0,0 +1,375 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library +from dataclasses import dataclass +import datetime +import io +import json +import logging +import sys +import traceback + +## Installed +from freezegun import freeze_time +import pytest + +## Application +import pythonjsonlogger +from pythonjsonlogger.core import RESERVED_ATTRS, BaseJsonFormatter +from pythonjsonlogger.json import JsonFormatter + +if pythonjsonlogger.ORJSON_AVAILABLE: + from pythonjsonlogger.orjson import OrjsonFormatter + + +### SETUP +### ============================================================================ +ALL_FORMATTERS: list[BaseJsonFormatter] = [JsonFormatter] +if pythonjsonlogger.ORJSON_AVAILABLE: + ALL_FORMATTERS.append(OrjsonFormatter) + +_LOGGER_COUNT = 0 + +@dataclass +class LoggingEnvironment: + logger: logging.Logger + buffer: io.StringIO + handler: logging.Handler + + def set_formatter(self, formatter: BaseJsonFormatter) -> None: + self.handler.setFormatter(formatter) + return + + def load_json(self) -> Any: + return json.loads(self.buffer.getvalue()) + + +@pytest.fixture +def env() -> LoggingEnvironment: + global _LOGGER_COUNT # pylint: disable=global-statement + _LOGGER_COUNT += 1 + logger = logging.getLogger(f"pythonjsonlogger.tests.{_LOGGER_COUNT}") + logger.setLevel(logging.DEBUG) + buffer = io.StringIO() + handler = logging.StreamHandler(buffer) + logger.addHandler(handler) + yield LoggingEnvironment(logger=logger, buffer=buffer, handler=handler) + logger.removeHandler(handler) + logger.setLevel(logging.NOTSET) + buffer.close() + return + + +def get_traceback_from_exception_followed_by_log_call(env_: LoggingEnvironment) -> str: + try: + raise Exception("test") + except Exception: + env_.logger.exception("hello") + str_traceback = traceback.format_exc() + # Formatter removes trailing new line + if str_traceback.endswith("\n"): + str_traceback = str_traceback[:-1] + return str_traceback + +### TESTS +### ============================================================================ +## Common Tests +## ----------------------------------------------------------------------------- +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_default_format(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_()) + + msg = "testing logging format" + env.logger.info(msg) + + log_json = env.load_json() + + assert log_json["message"] == msg + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_percentage_format(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_( + # All kind of different styles to check the regex + "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" + )) + + msg = "testing logging format" + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["message"] == msg + assert log_json.keys() == {"levelname", "message", "filename", "lineno", "asctime"} + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_rename_base_field(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_(rename_fields={"message": "@message"})) + + msg = "testing logging format" + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["@message"] == msg + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_rename_nonexistent_field(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_(rename_fields={"nonexistent_key": "new_name"})) + + stderr_watcher = io.StringIO() + sys.stderr = stderr_watcher + env.logger.info("testing logging rename") + sys.stderr == sys.__stderr__ + + assert "KeyError: 'nonexistent_key'" in stderr_watcher.getvalue() + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_add_static_fields(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_(static_fields={"log_stream": "kafka"})) + + msg = "testing static fields" + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["log_stream"] == "kafka" + assert log_json["message"] == msg + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_format_keys(env: LoggingEnvironment, class_: BaseJsonFormatter): + supported_keys = [ + "asctime", + "created", + "filename", + "funcName", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "message", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "thread", + "threadName", + ] + + log_format = lambda x: [f"%({i:s})s" for i in x] + custom_format = " ".join(log_format(supported_keys)) + + env.set_formatter(class_(custom_format)) + + msg = "testing logging format" + env.logger.info(msg) + log_json = env.load_json() + + for key in supported_keys: + assert key in log_json + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_unknown_format_key(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_("%(unknown_key)s %(message)s")) + env.logger.info("testing unknown logging format") + # make sure no error occurs + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_log_dict(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_()) + + msg = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["text"] == msg["text"] + assert log_json["num"] == msg["num"] + assert log_json["5"] == msg[5] + assert log_json["nested"] == msg["nested"] + assert log_json["message"] == "" + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_log_extra(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_()) + + extra = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} + env.logger.info("hello", extra=extra) + log_json = env.load_json() + + assert log_json["text"] == extra["text"] + assert log_json["num"] == extra["num"] + assert log_json["5"] == extra[5] + assert log_json["nested"] == extra["nested"] + assert log_json["message"] == "hello" + return + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: BaseJsonFormatter): + class CustomJsonFormatter(class_): + + def process_log_record(self, log_record): + log_record["custom"] = "value" + return super().process_log_record(log_record) + + env.set_formatter(CustomJsonFormatter()) + env.logger.info("message") + log_json = env.load_json() + + assert log_json["custom"] == "value" + return + + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_exc_info(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_()) + + expected_value = get_traceback_from_exception_followed_by_log_call(env) + log_json = env.load_json() + + assert log_json["exc_info"] == expected_value + return + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_exc_info_renamed(env: LoggingEnvironment, class_: BaseJsonFormatter): + env.set_formatter(class_("%(exc_info)s", rename_fields={"exc_info": "stack_trace"})) + + expected_value = get_traceback_from_exception_followed_by_log_call(env) + log_json = env.load_json() + + assert log_json["stack_trace"] == expected_value + assert "exc_info" not in log_json + return + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_custom_object_serialization(env: LoggingEnvironment, class_: BaseJsonFormatter): + def encode_complex(z): + if isinstance(z, complex): + return (z.real, z.imag) + raise TypeError(f"Object of type {type(z)} is no JSON serializable") + + env.set_formatter(class_(json_default=encode_complex)) + + env.logger.info("foo", extra={"special": complex(3, 8)}) + log_json = env.load_json() + + assert log_json["special"] == [3.0, 8.0] + return + +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatter): + log_format = lambda x: [f"%({i:s})s" for i in x] + reserved_attrs_map = { + "exc_info": "error.type", + "exc_text": "error.message", + "funcName": "log.origin.function", + "levelname": "log.level", + "module": "log.origin.file.name", + "processName": "process.name", + "threadName": "process.thread.name", + "msg": "log.message", + } + + custom_format = " ".join(log_format(reserved_attrs_map.keys())) + reserved_attrs = [ + attr for attr in RESERVED_ATTRS if attr not in list(reserved_attrs_map.keys()) + ] + env.set_formatter(class_( + custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map + )) + + env.logger.info("message") + log_json = env.load_json() + + # Note: this check is fragile if we make the following changes in the future (we might): + # - renaming fields no longer requires the field to be present (#6) + # - we add the ability (and data above) to rename a field to an existing field name + # e.g. {"exc_info": "trace_original", "@custom_trace": "exc_info"} + for old_name, new_name in reserved_attrs_map.items(): + assert new_name in log_json + assert old_name not in log_json + return + +@freeze_time(datetime.datetime(2017, 7, 14, 2, 40)) +@pytest.mark.parametrize("class_", ALL_FORMATTERS) +def test_json_default_encoder_with_timestamp(env: LoggingEnvironment, class_: BaseJsonFormatter): + if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter: + # https://github.com/spulec/freezegun/issues/448#issuecomment-1686070438 + pytest.xfail() + + env.set_formatter(class_(timestamp=True)) + + env.logger.info("Hello") + print(env.buffer.getvalue()) + log_json = env.load_json() + + assert log_json["timestamp"] == "2017-07-14T02:40:00+00:00" + return + +## JsonFormatter Specific +## ----------------------------------------------------------------------------- +def test_json_default_encoder(env: LoggingEnvironment): + env.set_formatter(JsonFormatter()) + + msg = { + "adate": datetime.datetime(1999, 12, 31, 23, 59), + "otherdate": datetime.date(1789, 7, 14), + "otherdatetime": datetime.datetime(1789, 7, 14, 23, 59), + "otherdatetimeagain": datetime.datetime(1900, 1, 1), + } + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["adate"] == "1999-12-31T23:59:00" + assert log_json["otherdate"] == "1789-07-14" + assert log_json["otherdatetime"] == "1789-07-14T23:59:00" + assert log_json["otherdatetimeagain"] == "1900-01-01T00:00:00" + return + +def test_json_custom_default(env: LoggingEnvironment): + def custom(o): + return "very custom" + + env.set_formatter(JsonFormatter(json_default=custom)) + + msg = {"adate": datetime.datetime(1999, 12, 31, 23, 59), "normal": "value"} + env.logger.info(msg) + log_json = env.load_json() + + assert log_json["adate"] == "very custom" + assert log_json["normal"] == "value" + return + +def test_ensure_ascii_true(env: LoggingEnvironment): + env.set_formatter(JsonFormatter()) + env.logger.info("Привет") + + # Note: we don't use env.load_json as we want to know the raw output + msg = env.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] + assert msg, r"\u041f\u0440\u0438\u0432\u0435\u0442" + return + +def test_ensure_ascii_false(env: LoggingEnvironment): + env.set_formatter(JsonFormatter(json_ensure_ascii=False)) + env.logger.info("Привет") + + # Note: we don't use env.load_json as we want to know the raw output + msg = env.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] + assert msg == "Привет" + return diff --git a/tests/test_jsonlogger.py b/tests/test_jsonlogger.py index e6805a0..7598dd7 100644 --- a/tests/test_jsonlogger.py +++ b/tests/test_jsonlogger.py @@ -13,289 +13,6 @@ class TestJsonLogger(unittest.TestCase): - def setUp(self): - self.log = logging.getLogger(f"logging-test-{random.randint(1, 101)}") - self.log.setLevel(logging.DEBUG) - self.buffer = StringIO() - - self.log_handler = logging.StreamHandler(self.buffer) - self.log.addHandler(self.log_handler) - - def test_default_format(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - - msg = "testing logging format" - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - - self.assertEqual(log_json["message"], msg) - - def test_percentage_format(self): - fr = JsonFormatter( - # All kind of different styles to check the regex - "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" - ) - self.log_handler.setFormatter(fr) - - msg = "testing logging format" - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - - self.assertEqual(log_json["message"], msg) - self.assertEqual(log_json.keys(), {"levelname", "message", "filename", "lineno", "asctime"}) - - def test_rename_base_field(self): - fr = JsonFormatter(rename_fields={"message": "@message"}) - self.log_handler.setFormatter(fr) - - msg = "testing logging format" - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - - self.assertEqual(log_json["@message"], msg) - - def test_rename_nonexistent_field(self): - fr = JsonFormatter(rename_fields={"nonexistent_key": "new_name"}) - self.log_handler.setFormatter(fr) - - stderr_watcher = StringIO() - sys.stderr = stderr_watcher - self.log.info("testing logging rename") - - self.assertTrue("KeyError: 'nonexistent_key'" in stderr_watcher.getvalue()) - - def test_add_static_fields(self): - fr = JsonFormatter(static_fields={"log_stream": "kafka"}) - - self.log_handler.setFormatter(fr) - - msg = "testing static fields" - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - - self.assertEqual(log_json["log_stream"], "kafka") - self.assertEqual(log_json["message"], msg) - - def test_format_keys(self): - supported_keys = [ - "asctime", - "created", - "filename", - "funcName", - "levelname", - "levelno", - "lineno", - "module", - "msecs", - "message", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "thread", - "threadName", - ] - - log_format = lambda x: [f"%({i:s})s" for i in x] - custom_format = " ".join(log_format(supported_keys)) - - fr = JsonFormatter(custom_format) - self.log_handler.setFormatter(fr) - - msg = "testing logging format" - self.log.info(msg) - log_msg = self.buffer.getvalue() - log_json = json.loads(log_msg) - - for supported_key in supported_keys: - if supported_key in log_json: - self.assertTrue(True) - - def test_unknown_format_key(self): - fr = JsonFormatter("%(unknown_key)s %(message)s") - - self.log_handler.setFormatter(fr) - msg = "testing unknown logging format" - try: - self.log.info(msg) - except Exception: - self.assertTrue(False, "Should succeed") - - def test_log_adict(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - - msg = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} - - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("text"), msg["text"]) - self.assertEqual(log_json.get("num"), msg["num"]) - self.assertEqual(log_json.get("5"), msg[5]) - self.assertEqual(log_json.get("nested"), msg["nested"]) - self.assertEqual(log_json["message"], "") - - def test_log_extra(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - - extra = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} - self.log.info("hello", extra=extra) - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("text"), extra["text"]) - self.assertEqual(log_json.get("num"), extra["num"]) - self.assertEqual(log_json.get("5"), extra[5]) - self.assertEqual(log_json.get("nested"), extra["nested"]) - self.assertEqual(log_json["message"], "hello") - - def test_json_default_encoder(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - - msg = { - "adate": datetime.datetime(1999, 12, 31, 23, 59), - "otherdate": datetime.date(1789, 7, 14), - "otherdatetime": datetime.datetime(1789, 7, 14, 23, 59), - "otherdatetimeagain": datetime.datetime(1900, 1, 1), - } - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("adate"), "1999-12-31T23:59:00") - self.assertEqual(log_json.get("otherdate"), "1789-07-14") - self.assertEqual(log_json.get("otherdatetime"), "1789-07-14T23:59:00") - self.assertEqual(log_json.get("otherdatetimeagain"), "1900-01-01T00:00:00") - - @unittest.mock.patch("time.time", return_value=1500000000.0) - def test_json_default_encoder_with_timestamp(self, time_mock): - fr = JsonFormatter(timestamp=True) - self.log_handler.setFormatter(fr) - - self.log.info("Hello") - - self.assertTrue(time_mock.called) - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("timestamp"), "2017-07-14T02:40:00+00:00") - - def test_json_custom_default(self): - def custom(o): - return "very custom" - - fr = JsonFormatter(json_default=custom) - self.log_handler.setFormatter(fr) - - msg = {"adate": datetime.datetime(1999, 12, 31, 23, 59), "normal": "value"} - self.log.info(msg) - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("adate"), "very custom") - self.assertEqual(log_json.get("normal"), "value") - - def test_json_custom_logic_adds_field(self): - class CustomJsonFormatter(JsonFormatter): - - def process_log_record(self, log_record): - log_record["custom"] = "value" - # Old Style "super" since Python 2.6's logging.Formatter is old - # style - return JsonFormatter.process_log_record(self, log_record) - - self.log_handler.setFormatter(CustomJsonFormatter()) - self.log.info("message") - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("custom"), "value") - - def get_traceback_from_exception_followed_by_log_call(self) -> str: - try: - raise Exception("test") - except Exception: - self.log.exception("hello") - str_traceback = traceback.format_exc() - # Formatter removes trailing new line - if str_traceback.endswith("\n"): - str_traceback = str_traceback[:-1] - - return str_traceback - - def test_exc_info(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - expected_value = self.get_traceback_from_exception_followed_by_log_call() - - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("exc_info"), expected_value) - - def test_exc_info_renamed(self): - fr = JsonFormatter("%(exc_info)s", rename_fields={"exc_info": "stack_trace"}) - self.log_handler.setFormatter(fr) - expected_value = self.get_traceback_from_exception_followed_by_log_call() - - log_json = json.loads(self.buffer.getvalue()) - self.assertEqual(log_json.get("stack_trace"), expected_value) - self.assertEqual(log_json.get("exc_info"), None) - - def test_ensure_ascii_true(self): - fr = JsonFormatter() - self.log_handler.setFormatter(fr) - self.log.info("Привет") - msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] - self.assertEqual(msg, r"\u041f\u0440\u0438\u0432\u0435\u0442") - - def test_ensure_ascii_false(self): - fr = JsonFormatter(json_ensure_ascii=False) - self.log_handler.setFormatter(fr) - self.log.info("Привет") - msg = self.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] - self.assertEqual(msg, "Привет") - - def test_custom_object_serialization(self): - def encode_complex(z): - if isinstance(z, complex): - return (z.real, z.imag) - else: - type_name = z.__class__.__name__ - raise TypeError(f"Object of type '{type_name}' is no JSON serializable") - - formatter = JsonFormatter(json_default=encode_complex, json_encoder=json.JSONEncoder) - self.log_handler.setFormatter(formatter) - - value = { - "special": complex(3, 8), - } - - self.log.info(" message", extra=value) - msg = self.buffer.getvalue() - self.assertEqual(msg, '{"message": " message", "special": [3.0, 8.0]}\n') - - def test_rename_reserved_attrs(self): - log_format = lambda x: [f"%({i:s})s" for i in x] - reserved_attrs_map = { - "exc_info": "error.type", - "exc_text": "error.message", - "funcName": "log.origin.function", - "levelname": "log.level", - "module": "log.origin.file.name", - "processName": "process.name", - "threadName": "process.thread.name", - "msg": "log.message", - } - - custom_format = " ".join(log_format(reserved_attrs_map.keys())) - reserved_attrs = [ - attr for attr in RESERVED_ATTRS if attr not in list(reserved_attrs_map.keys()) - ] - formatter = JsonFormatter( - custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map - ) - self.log_handler.setFormatter(formatter) - self.log.info("message") - - msg = self.buffer.getvalue() - self.assertEqual( - msg, - '{"error.type": null, "error.message": null, "log.origin.function": "test_rename_reserved_attrs", "log.level": "INFO", "log.origin.file.name": "test_jsonlogger", "process.name": "MainProcess", "process.thread.name": "MainThread", "log.message": "message"}\n', - ) - def test_merge_record_extra(self): record = logging.LogRecord( "name", level=1, pathname="", lineno=1, msg="Some message", args=None, exc_info=None From c017befe85f5221751538f7b794b435fbbc31138 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 28 Apr 2024 14:26:07 +1000 Subject: [PATCH 12/27] Update docstrings to use mkdocstrings compatible --- src/pythonjsonlogger/core.py | 36 ++++++++++++++++++++++++++++-------- src/pythonjsonlogger/json.py | 32 ++++++++++++++------------------ 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 0965864..722d00b 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -96,11 +96,12 @@ def merge_record_extra( """ Merges extra attributes from LogRecord object into target dictionary - :param record: logging.LogRecord - :param target: dict to update - :param reserved: dict or list with reserved keys to skip - :param rename_fields: an optional dict, used to rename field names in the output. - Rename levelname to log.level: {'levelname': 'log.level'} + Args: + record: logging.LogRecord + target: dict to update + reserved: dict or list with reserved keys to skip + rename_fields: an optional dict, used to rename field names in the output. + e.g. Rename `levelname` to `log.level`: `{'levelname': 'log.level'}` *Changed in 4.0*: `reserved` is now `Container[str]`. """ @@ -199,7 +200,11 @@ def __init__( return def format(self, record: logging.LogRecord) -> str: - """Formats a log record and serializes to json""" + """Formats a log record and serializes to json + + Args: + record: the record to format + """ message_dict: Dict[str, Any] = {} # TODO: logging.LogRecord.msg and logging.LogRecord.message in typeshed # are always type of str. We shouldn't need to override that. @@ -238,6 +243,9 @@ def parse(self) -> List[str]: to include in all log messages. You can support custom styles by overriding this method. + + Returns: + list of fields to be extracted and serialized """ if isinstance(self._style, logging.StringTemplateStyle): formatter_style_pattern = STYLE_STRING_TEMPLATE_REGEX @@ -272,8 +280,14 @@ def add_fields( record: logging.LogRecord, message_dict: Dict[str, Any], ) -> None: - """ - Override this method to implement custom logic for adding fields. + """Extract fields from a LogRecord for logging + + This method can be overridden to implement custom logic for adding fields. + + Args: + log_record: data that will be logged + record: the record to extract data from + message_dict: ??? """ for field in self._required_fields: log_record[field] = record.__dict__.get(field) @@ -308,6 +322,9 @@ def jsonify_log_record(self, log_record: LogRecord) -> str: """Convert this log record into a JSON string. Child classes MUST override this method. + + Args: + log_record: the data to serialize """ raise NotImplementedError() @@ -316,5 +333,8 @@ def process_log_record(self, log_record: LogRecord) -> LogRecord: Child classes can override this method to alter the log record before it is serialized. + + Args: + log_record: incoming data """ return log_record diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py index e248347..a7f82f4 100644 --- a/src/pythonjsonlogger/json.py +++ b/src/pythonjsonlogger/json.py @@ -1,5 +1,8 @@ -"""JSON Formatter using the standard library `json` module""" +"""JSON formatter using the standard library's `json` for encoding. +Module contains the `JsonFormatter` and a custom `JsonEncoder` which supports a greater +variety of types. +""" ### IMPORTS ### ============================================================================ ## Future @@ -19,9 +22,7 @@ ### CLASSES ### ============================================================================ class JsonEncoder(json.JSONEncoder): - """ - A custom encoder extending the default JSONEncoder - """ + """A custom encoder extending the default JSONEncoder""" def default(self, o: Any) -> Any: if isinstance(o, (date, datetime, time)): @@ -54,13 +55,8 @@ def format_datetime_obj(self, o): class JsonFormatter(core.BaseJsonFormatter): - """ - A custom formatter to format logging records as json strings. - Extra values will be formatted as str() if not supported by - json default encoder - """ + """JSON formatter using the standard library's `json` for encoding""" - # pylint: disable=too-many-arguments def __init__( self, *args, @@ -72,13 +68,14 @@ def __init__( **kwargs, ) -> None: """ - :param json_default: a function for encoding non-standard objects - as outlined in https://docs.python.org/3/library/json.html - :param json_encoder: optional custom encoder - :param json_serializer: a :meth:`json.dumps`-compatible callable - that will be used to serialize the log record. - :param json_indent: indent parameter for json.dumps - :param json_ensure_ascii: ensure_ascii parameter for json.dumps + Args: + json_default: a function for encoding non-standard objects + as outlined in https://docs.python.org/3/library/json.html + json_encoder: optional custom encoder + json_serializer: a :meth:`json.dumps`-compatible callable + that will be used to serialize the log record. + json_indent: indent parameter for json.dumps + json_ensure_ascii: ensure_ascii parameter for json.dumps """ super().__init__(*args, **kwargs) @@ -92,7 +89,6 @@ def __init__( return def jsonify_log_record(self, log_record: core.LogRecord) -> str: - """Returns a json string of the log record.""" return self.json_serializer( log_record, default=self.json_default, From 457a74c3f5a6f81b63ba7135fcf7f1c525187c47 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 28 Apr 2024 14:34:42 +1000 Subject: [PATCH 13/27] Fix formatting, linting, typing --- src/pythonjsonlogger/__init__.py | 1 + src/pythonjsonlogger/json.py | 1 + tests/test_formatters.py | 72 +++++++++++++++++++------------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index c43dab1..187c0c6 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -18,6 +18,7 @@ else: try: import orjson + ORJSON_AVAILABLE = True except ImportError: ORJSON_AVAILABLE = False diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py index a7f82f4..d9c4110 100644 --- a/src/pythonjsonlogger/json.py +++ b/src/pythonjsonlogger/json.py @@ -3,6 +3,7 @@ Module contains the `JsonFormatter` and a custom `JsonEncoder` which supports a greater variety of types. """ + ### IMPORTS ### ============================================================================ ## Future diff --git a/tests/test_formatters.py b/tests/test_formatters.py index a9d06a1..2c281b4 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -11,6 +11,7 @@ import logging import sys import traceback +from typing import Any, Generator ## Installed from freezegun import freeze_time @@ -27,12 +28,13 @@ ### SETUP ### ============================================================================ -ALL_FORMATTERS: list[BaseJsonFormatter] = [JsonFormatter] +ALL_FORMATTERS: list[type[BaseJsonFormatter]] = [JsonFormatter] if pythonjsonlogger.ORJSON_AVAILABLE: ALL_FORMATTERS.append(OrjsonFormatter) _LOGGER_COUNT = 0 + @dataclass class LoggingEnvironment: logger: logging.Logger @@ -48,7 +50,7 @@ def load_json(self) -> Any: @pytest.fixture -def env() -> LoggingEnvironment: +def env() -> Generator[LoggingEnvironment, None, None]: global _LOGGER_COUNT # pylint: disable=global-statement _LOGGER_COUNT += 1 logger = logging.getLogger(f"pythonjsonlogger.tests.{_LOGGER_COUNT}") @@ -74,12 +76,13 @@ def get_traceback_from_exception_followed_by_log_call(env_: LoggingEnvironment) str_traceback = str_traceback[:-1] return str_traceback + ### TESTS ### ============================================================================ ## Common Tests ## ----------------------------------------------------------------------------- @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_default_format(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_default_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_()) msg = "testing logging format" @@ -92,11 +95,13 @@ def test_default_format(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_percentage_format(env: LoggingEnvironment, class_: BaseJsonFormatter): - env.set_formatter(class_( - # All kind of different styles to check the regex - "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" - )) +def test_percentage_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): + env.set_formatter( + class_( + # All kind of different styles to check the regex + "[%(levelname)8s] %(message)s %(filename)s:%(lineno)d %(asctime)" + ) + ) msg = "testing logging format" env.logger.info(msg) @@ -108,7 +113,7 @@ def test_percentage_format(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_rename_base_field(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_rename_base_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(rename_fields={"message": "@message"})) msg = "testing logging format" @@ -120,7 +125,7 @@ def test_rename_base_field(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_rename_nonexistent_field(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_rename_nonexistent_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(rename_fields={"nonexistent_key": "new_name"})) stderr_watcher = io.StringIO() @@ -133,7 +138,7 @@ def test_rename_nonexistent_field(env: LoggingEnvironment, class_: BaseJsonForma @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_add_static_fields(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_add_static_fields(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_(static_fields={"log_stream": "kafka"})) msg = "testing static fields" @@ -146,7 +151,7 @@ def test_add_static_fields(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_format_keys(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_format_keys(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): supported_keys = [ "asctime", "created", @@ -182,7 +187,7 @@ def test_format_keys(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_unknown_format_key(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_unknown_format_key(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_("%(unknown_key)s %(message)s")) env.logger.info("testing unknown logging format") # make sure no error occurs @@ -190,7 +195,7 @@ def test_unknown_format_key(env: LoggingEnvironment, class_: BaseJsonFormatter): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_log_dict(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_log_dict(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_()) msg = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} @@ -199,18 +204,18 @@ def test_log_dict(env: LoggingEnvironment, class_: BaseJsonFormatter): assert log_json["text"] == msg["text"] assert log_json["num"] == msg["num"] - assert log_json["5"] == msg[5] + assert log_json["5"] == msg[5] assert log_json["nested"] == msg["nested"] assert log_json["message"] == "" return @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_log_extra(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_log_extra(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_()) extra = {"text": "testing logging", "num": 1, 5: "9", "nested": {"more": "data"}} - env.logger.info("hello", extra=extra) + env.logger.info("hello", extra=extra) # type: ignore[arg-type] log_json = env.load_json() assert log_json["text"] == extra["text"] @@ -220,9 +225,10 @@ def test_log_extra(env: LoggingEnvironment, class_: BaseJsonFormatter): assert log_json["message"] == "hello" return + @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: BaseJsonFormatter): - class CustomJsonFormatter(class_): +def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): + class CustomJsonFormatter(class_): # type: ignore[valid-type,misc] def process_log_record(self, log_record): log_record["custom"] = "value" @@ -237,7 +243,7 @@ def process_log_record(self, log_record): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_exc_info(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_exc_info(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_()) expected_value = get_traceback_from_exception_followed_by_log_call(env) @@ -246,8 +252,9 @@ def test_exc_info(env: LoggingEnvironment, class_: BaseJsonFormatter): assert log_json["exc_info"] == expected_value return + @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_exc_info_renamed(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_exc_info_renamed(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): env.set_formatter(class_("%(exc_info)s", rename_fields={"exc_info": "stack_trace"})) expected_value = get_traceback_from_exception_followed_by_log_call(env) @@ -257,14 +264,15 @@ def test_exc_info_renamed(env: LoggingEnvironment, class_: BaseJsonFormatter): assert "exc_info" not in log_json return + @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_custom_object_serialization(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_custom_object_serialization(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): def encode_complex(z): if isinstance(z, complex): return (z.real, z.imag) raise TypeError(f"Object of type {type(z)} is no JSON serializable") - env.set_formatter(class_(json_default=encode_complex)) + env.set_formatter(class_(json_default=encode_complex)) # type: ignore[call-arg] env.logger.info("foo", extra={"special": complex(3, 8)}) log_json = env.load_json() @@ -272,8 +280,9 @@ def encode_complex(z): assert log_json["special"] == [3.0, 8.0] return + @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_rename_reserved_attrs(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): log_format = lambda x: [f"%({i:s})s" for i in x] reserved_attrs_map = { "exc_info": "error.type", @@ -290,9 +299,9 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatte reserved_attrs = [ attr for attr in RESERVED_ATTRS if attr not in list(reserved_attrs_map.keys()) ] - env.set_formatter(class_( - custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map - )) + env.set_formatter( + class_(custom_format, reserved_attrs=reserved_attrs, rename_fields=reserved_attrs_map) + ) env.logger.info("message") log_json = env.load_json() @@ -306,9 +315,12 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: BaseJsonFormatte assert old_name not in log_json return + @freeze_time(datetime.datetime(2017, 7, 14, 2, 40)) @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_json_default_encoder_with_timestamp(env: LoggingEnvironment, class_: BaseJsonFormatter): +def test_json_default_encoder_with_timestamp( + env: LoggingEnvironment, class_: type[BaseJsonFormatter] +): if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter: # https://github.com/spulec/freezegun/issues/448#issuecomment-1686070438 pytest.xfail() @@ -322,6 +334,7 @@ def test_json_default_encoder_with_timestamp(env: LoggingEnvironment, class_: Ba assert log_json["timestamp"] == "2017-07-14T02:40:00+00:00" return + ## JsonFormatter Specific ## ----------------------------------------------------------------------------- def test_json_default_encoder(env: LoggingEnvironment): @@ -342,6 +355,7 @@ def test_json_default_encoder(env: LoggingEnvironment): assert log_json["otherdatetimeagain"] == "1900-01-01T00:00:00" return + def test_json_custom_default(env: LoggingEnvironment): def custom(o): return "very custom" @@ -356,6 +370,7 @@ def custom(o): assert log_json["normal"] == "value" return + def test_ensure_ascii_true(env: LoggingEnvironment): env.set_formatter(JsonFormatter()) env.logger.info("Привет") @@ -365,6 +380,7 @@ def test_ensure_ascii_true(env: LoggingEnvironment): assert msg, r"\u041f\u0440\u0438\u0432\u0435\u0442" return + def test_ensure_ascii_false(env: LoggingEnvironment): env.set_formatter(JsonFormatter(json_ensure_ascii=False)) env.logger.info("Привет") From 0d2bc6504e942ae1eab572a9a572593100c380e6 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 28 Apr 2024 14:48:50 +1000 Subject: [PATCH 14/27] Remove py37 from tox config --- tox.ini | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index db04dd8..31ce42b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,12 @@ [tox] requires = tox>=3 -envlist = lint, type, pypy{37,38,39,310}, py{37,38,39,310,311,312,313} +envlist = lint, type, pypy{38,39,310}, py{38,39,310,311,312,313} [gh-actions] python = - pypy-3.7: pypy37 pypy-3.8: pypy38 pypy-3.9: pypy39 pypy-3.10: pypy310 - 3.7: py37 3.8: py38 3.9: py39 3.10: py310 From 0ee6b9adbc7c5481dae8f9fe71b71beba0db8e12 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Tue, 30 Apr 2024 08:18:50 +1000 Subject: [PATCH 15/27] Maintain backwards compatibility --- pyproject.toml | 2 +- src/pythonjsonlogger/__init__.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63abcb5..6538187 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "4.0.0.dev1" +version = "3.1.0.dev1" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index 187c0c6..981e1c7 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -4,10 +4,12 @@ ## Standard Library import sys +import warnings ## Installed ## Application +import pythonjsonlogger.json ### CONSTANTS ### ============================================================================ @@ -22,3 +24,15 @@ ORJSON_AVAILABLE = True except ImportError: ORJSON_AVAILABLE = False + + +### DEPRECATED COMPATIBILITY +### ============================================================================ +def __getattr__(name: str): + if name == "jsonlogger": + warnings.warn( + "pythonjsonlogger.jsonlogger has been moved to pythonjsonlogger.json", + DeprecationWarning, + ) + return pythonjsonlogger.json + raise AttributeError(f"module {__name__} has no attribute {name}") From a0b595bd2843d5b1fde2804a3648a2862a502d64 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Tue, 30 Apr 2024 17:06:35 +1000 Subject: [PATCH 16/27] Add more tests --- tests/test_deprecation.py | 20 ++++++++++++++++++++ tests/test_formatters.py | 14 ++++++++++++-- tests/test_jsonlogger.py | 24 ------------------------ 3 files changed, 32 insertions(+), 26 deletions(-) create mode 100644 tests/test_deprecation.py delete mode 100644 tests/test_jsonlogger.py diff --git a/tests/test_deprecation.py b/tests/test_deprecation.py new file mode 100644 index 0000000..cc58783 --- /dev/null +++ b/tests/test_deprecation.py @@ -0,0 +1,20 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library + +## Installed +import pytest + +## Application +import pythonjsonlogger + + +### TESTS +### ============================================================================ +def test_jsonlogger_deprecated(): + with pytest.deprecated_call(): + pythonjsonlogger.jsonlogger + return diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 2c281b4..744da45 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -19,7 +19,7 @@ ## Application import pythonjsonlogger -from pythonjsonlogger.core import RESERVED_ATTRS, BaseJsonFormatter +from pythonjsonlogger.core import RESERVED_ATTRS, BaseJsonFormatter, merge_record_extra from pythonjsonlogger.json import JsonFormatter if pythonjsonlogger.ORJSON_AVAILABLE: @@ -79,7 +79,17 @@ def get_traceback_from_exception_followed_by_log_call(env_: LoggingEnvironment) ### TESTS ### ============================================================================ -## Common Tests +def test_merge_record_extra(): + record = logging.LogRecord( + "name", level=1, pathname="", lineno=1, msg="Some message", args=None, exc_info=None + ) + output = merge_record_extra(record, target={"foo": "bar"}, reserved=[]) + assert output["foo"] == "bar" + assert output["msg"] == "Some message" + return + + +## Common Formatter Tests ## ----------------------------------------------------------------------------- @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_default_format(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): diff --git a/tests/test_jsonlogger.py b/tests/test_jsonlogger.py deleted file mode 100644 index 7598dd7..0000000 --- a/tests/test_jsonlogger.py +++ /dev/null @@ -1,24 +0,0 @@ -import datetime -import logging -from io import StringIO -import json -import random -import sys -import traceback -import unittest -import unittest.mock - -from pythonjsonlogger.core import RESERVED_ATTRS, merge_record_extra -from pythonjsonlogger.json import JsonFormatter - - -class TestJsonLogger(unittest.TestCase): - def test_merge_record_extra(self): - record = logging.LogRecord( - "name", level=1, pathname="", lineno=1, msg="Some message", args=None, exc_info=None - ) - output = merge_record_extra(record, target=dict(foo="bar"), reserved=[]) - self.assertIn("foo", output) - self.assertIn("msg", output) - self.assertEqual(output["foo"], "bar") - self.assertEqual(output["msg"], "Some message") From 77dcdae5974424b5f44ae11b65525bea92909a42 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Wed, 1 May 2024 08:29:52 +1000 Subject: [PATCH 17/27] Add support for deprecated json.RESERVED_ATTRS --- src/pythonjsonlogger/json.py | 13 +++++++++++++ tests/test_deprecation.py | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py index d9c4110..a64c549 100644 --- a/src/pythonjsonlogger/json.py +++ b/src/pythonjsonlogger/json.py @@ -15,6 +15,7 @@ import json import traceback from typing import Any, Callable, Optional, Union +import warnings ## Application from . import core @@ -97,3 +98,15 @@ def jsonify_log_record(self, log_record: core.LogRecord) -> str: indent=self.json_indent, ensure_ascii=self.json_ensure_ascii, ) + + +### DEPRECATED COMPATIBILITY +### ============================================================================ +def __getattr__(name: str): + if name == "RESERVED_ATTRS": + warnings.warn( + "RESERVED_ATTRS has been moved to pythonjsonlogger.core", + DeprecationWarning, + ) + return core.RESERVED_ATTRS + raise AttributeError(f"module {__name__} has no attribute {name}") diff --git a/tests/test_deprecation.py b/tests/test_deprecation.py index cc58783..ad4c988 100644 --- a/tests/test_deprecation.py +++ b/tests/test_deprecation.py @@ -18,3 +18,11 @@ def test_jsonlogger_deprecated(): with pytest.deprecated_call(): pythonjsonlogger.jsonlogger return + + +def test_jsonlogger_reserved_attrs_deprecated(): + with pytest.deprecated_call(): + # Note: We use json instead of jsonlogger as jsonlogger will also produce + # a DeprecationWarning and we specifically want the one for RESERVED_ATTRS + pythonjsonlogger.json.RESERVED_ATTRS + return From b61fc986b70d6ed75fa2b89da6d26e09415cdf8e Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Wed, 1 May 2024 16:41:25 +1000 Subject: [PATCH 18/27] fix assert in test --- tests/test_formatters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 744da45..070c2f5 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -387,7 +387,7 @@ def test_ensure_ascii_true(env: LoggingEnvironment): # Note: we don't use env.load_json as we want to know the raw output msg = env.buffer.getvalue().split('"message": "', 1)[1].split('"', 1)[0] - assert msg, r"\u041f\u0440\u0438\u0432\u0435\u0442" + assert msg == r"\u041f\u0440\u0438\u0432\u0435\u0442" return From 89f08202b235768b135ceec2590705a22f31d2ad Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 17:57:02 +1000 Subject: [PATCH 19/27] Update test names --- tests/test_formatters.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 070c2f5..cde77c0 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -237,7 +237,7 @@ def test_log_extra(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_json_custom_logic_adds_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): +def test_custom_logic_adds_field(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): class CustomJsonFormatter(class_): # type: ignore[valid-type,misc] def process_log_record(self, log_record): @@ -328,7 +328,7 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: type[BaseJsonFor @freeze_time(datetime.datetime(2017, 7, 14, 2, 40)) @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_json_default_encoder_with_timestamp( +def test_default_encoder_with_timestamp( env: LoggingEnvironment, class_: type[BaseJsonFormatter] ): if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter: @@ -381,7 +381,7 @@ def custom(o): return -def test_ensure_ascii_true(env: LoggingEnvironment): +def test_json_ensure_ascii_true(env: LoggingEnvironment): env.set_formatter(JsonFormatter()) env.logger.info("Привет") @@ -391,7 +391,7 @@ def test_ensure_ascii_true(env: LoggingEnvironment): return -def test_ensure_ascii_false(env: LoggingEnvironment): +def test_json_ensure_ascii_false(env: LoggingEnvironment): env.set_formatter(JsonFormatter(json_ensure_ascii=False)) env.logger.info("Привет") From a2df91b71baf275d4933b18b771271945d08f6ae Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 18:09:09 +1000 Subject: [PATCH 20/27] Add support for msgspec --- pyproject.toml | 5 ++++ src/pythonjsonlogger/__init__.py | 8 ++++++ src/pythonjsonlogger/json.py | 6 ++++- src/pythonjsonlogger/msgspec.py | 43 ++++++++++++++++++++++++++++++++ src/pythonjsonlogger/orjson.py | 6 ++++- tests/test_formatters.py | 11 +++++--- 6 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 src/pythonjsonlogger/msgspec.py diff --git a/pyproject.toml b/pyproject.toml index 6538187..863b557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,9 +47,14 @@ orjson = [ "orjson;implementation_name!='pypy'", ] +msgspec = [ + "msgspec", +] + dev = [ ## Optional but required for dev "orjson;implementation_name!='pypy' and python_version<'3.13'", + "msgspec", ## Lint "validate-pyproject[all]", "black", diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index 981e1c7..e03cbbd 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -26,6 +26,14 @@ ORJSON_AVAILABLE = False +try: + import msgspec + + MSGSPEC_AVAILABLE = True +except ImportError: + MSGSPEC_AVAILABLE = False + + ### DEPRECATED COMPATIBILITY ### ============================================================================ def __getattr__(name: str): diff --git a/src/pythonjsonlogger/json.py b/src/pythonjsonlogger/json.py index a64c549..9d14a1d 100644 --- a/src/pythonjsonlogger/json.py +++ b/src/pythonjsonlogger/json.py @@ -24,7 +24,11 @@ ### CLASSES ### ============================================================================ class JsonEncoder(json.JSONEncoder): - """A custom encoder extending the default JSONEncoder""" + """A custom encoder extending the default JSONEncoder + + Refs: + - https://docs.python.org/3/library/json.html + """ def default(self, o: Any) -> Any: if isinstance(o, (date, datetime, time)): diff --git a/src/pythonjsonlogger/msgspec.py b/src/pythonjsonlogger/msgspec.py new file mode 100644 index 0000000..e711224 --- /dev/null +++ b/src/pythonjsonlogger/msgspec.py @@ -0,0 +1,43 @@ +### IMPORTS +### ============================================================================ +## Future +from __future__ import annotations + +## Standard Library + +## Installed +import msgspec.json + +## Application +from . import core + + +### CLASSES +### ============================================================================ +class MsgspecFormatter(core.BaseJsonFormatter): + """JSON formatter using msgspec.json for encoding. + + Refs: + - https://jcristharif.com/msgspec/api.html#msgspec.json.Encoder + """ + + # pylint: disable=too-many-arguments + def __init__( + self, + *args, + json_default: core.OptionalCallableOrStr = None, + **kwargs, + ) -> None: + """ + Args: + json_default: a function for encoding non-standard objects see: `msgspec.json.Encode:enc_hook` + """ + super().__init__(*args, **kwargs) + + self.json_default = core.str_to_object(json_default) + self._encoder = msgspec.json.Encoder(enc_hook=self.json_default) + return + + def jsonify_log_record(self, log_record: core.LogRecord) -> str: + """Returns a json string of the log record.""" + return self._encoder.encode(log_record).decode("utf8") diff --git a/src/pythonjsonlogger/orjson.py b/src/pythonjsonlogger/orjson.py index b6317c3..4c5dbab 100644 --- a/src/pythonjsonlogger/orjson.py +++ b/src/pythonjsonlogger/orjson.py @@ -15,7 +15,11 @@ ### CLASSES ### ============================================================================ class OrjsonFormatter(core.BaseJsonFormatter): - """JSON formatter using orjson for encoding.""" + """JSON formatter using orjson for encoding. + + Refs: + - https://github.com/ijl/orjson + """ # pylint: disable=too-many-arguments def __init__( diff --git a/tests/test_formatters.py b/tests/test_formatters.py index cde77c0..c80dd6f 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -25,12 +25,16 @@ if pythonjsonlogger.ORJSON_AVAILABLE: from pythonjsonlogger.orjson import OrjsonFormatter +if pythonjsonlogger.MSGSPEC_AVAILABLE: + from pythonjsonlogger.msgspec import MsgspecFormatter ### SETUP ### ============================================================================ ALL_FORMATTERS: list[type[BaseJsonFormatter]] = [JsonFormatter] if pythonjsonlogger.ORJSON_AVAILABLE: ALL_FORMATTERS.append(OrjsonFormatter) +if pythonjsonlogger.MSGSPEC_AVAILABLE: + ALL_FORMATTERS.append(MsgspecFormatter) _LOGGER_COUNT = 0 @@ -328,13 +332,14 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: type[BaseJsonFor @freeze_time(datetime.datetime(2017, 7, 14, 2, 40)) @pytest.mark.parametrize("class_", ALL_FORMATTERS) -def test_default_encoder_with_timestamp( - env: LoggingEnvironment, class_: type[BaseJsonFormatter] -): +def test_default_encoder_with_timestamp(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter: # https://github.com/spulec/freezegun/issues/448#issuecomment-1686070438 pytest.xfail() + if pythonjsonlogger.MSGSPEC_AVAILABLE and class_ is MsgspecFormatter: + pytest.xfail() + env.set_formatter(class_(timestamp=True)) env.logger.info("Hello") From 44c610d3f9509d8b2723d07fc88e782042d63873 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 18:13:48 +1000 Subject: [PATCH 21/27] Fix msgspec specifiers --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 863b557..c2b1b8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,13 +48,13 @@ orjson = [ ] msgspec = [ - "msgspec", + "msgspec;implementation_name!='pypy'", ] dev = [ ## Optional but required for dev "orjson;implementation_name!='pypy' and python_version<'3.13'", - "msgspec", + "msgspec;implementation_name!='pypy' and python_version<'3.13'", ## Lint "validate-pyproject[all]", "black", From 32f763fd2c243a1e8c9dab756c7bb83810a75dcc Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 18:19:58 +1000 Subject: [PATCH 22/27] simplify ORJSON_AVAILABLE check --- src/pythonjsonlogger/__init__.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/pythonjsonlogger/__init__.py b/src/pythonjsonlogger/__init__.py index e03cbbd..ed3ae60 100644 --- a/src/pythonjsonlogger/__init__.py +++ b/src/pythonjsonlogger/__init__.py @@ -3,7 +3,6 @@ ## Future ## Standard Library -import sys import warnings ## Installed @@ -13,17 +12,12 @@ ### CONSTANTS ### ============================================================================ -if sys.implementation.name == "pypy": - # Per https://github.com/ijl/orjson (last checked 2024-04-28) - # > orjson does not and will not support PyPy - ORJSON_AVAILABLE = False -else: - try: - import orjson +try: + import orjson - ORJSON_AVAILABLE = True - except ImportError: - ORJSON_AVAILABLE = False + ORJSON_AVAILABLE = True +except ImportError: + ORJSON_AVAILABLE = False try: From 487388cb2a11c5a5c173f076d8dd155b3e87dd00 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 19:17:46 +1000 Subject: [PATCH 23/27] Update README and CHANGELOG --- CHANGELOG.md | 27 ++++++++++++++ README.md | 72 +++++++++++++++++++++--------------- pyproject.toml | 2 +- src/pythonjsonlogger/core.py | 12 +++--- 4 files changed, 78 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33a0ca9..9cc7dd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,33 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.1.0.rc1](https://github.com/nhairs/python-json-logger/compare/v3.0.1...v3.1.0.rc1) - 2023-05-03 + +This splits common funcitonality out to allow supporting other JSON encoders. Although this is a large refactor, backwards compatibility has been maintained. + +### Added +- `.core` - more details below. +- Orjson encoder support via `.orjson.OrjsonFormatter`. +- MsgSpec encoder support via `.msgspec.MsgspecFormatter`. + +### Changed +- `.jsonlogger` has been moved to `.json` with core functionality moved to `.core`. +- `.core.BaseJsonFormatter` properly supports all `logging.Formatter` arguments: + - `fmt` is unchanged. + - `datefmt` is unchanged. + - `style` can now support non-standard arguments by setting `validate` to `False` + - `validate` allows non-standard `style` arguments or prevents calling `validate` on standard `style` arguments. + - `default` is ignored. + +### Deprecated +- `.jsonlogger` is now `.json` +- `.jsonlogger.RESERVED_ATTRS` is now `.core.RESERVED_ATTRS`. +- `.jsonlogger.merge_record_extra` is now `.core.merge_record_extra`. + +### Removed +- Python 3.7 support dropped +- `.jsonlogger.JsonFormatter._str_to_fn` replaced with `.core.str_to_object`. + ## [3.0.1](https://github.com/nhairs/python-json-logger/compare/v3.0.0...v3.0.1) - 2023-04-01 ### Fixes diff --git a/README.md b/README.md index c7369b9..f646c42 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Until the PEP 541 request is complete you will need to install directly from git To install from releases: ```shell -# 3.0.0 wheel +# e.g. 3.0.0 wheel pip install 'python-json-logger@https://github.com/nhairs/python-json-logger/releases/download/v3.0.0/python_json_logger-3.0.0-py3-none-any.whl' ``` @@ -53,38 +53,30 @@ pip install -e . ## Usage +Python JSON Logger provides `logging.Formatter`s that encode the logged message into JSON. Although a variety of JSON encoders are supported, in the following examples we will use the `pythonjsonlogger.json.JsonFormatter` which uses the the `json` module from the standard library. + ### Integrating with Python's logging framework -Json outputs are provided by the JsonFormatter logging formatter. You can add the custom formatter like below: +To produce JSON output, attach the formatter to a logging handler: ```python import logging - from pythonjsonlogger import jsonlogger + from pythonjsonlogger.json import JsonFormatter logger = logging.getLogger() logHandler = logging.StreamHandler() - formatter = jsonlogger.JsonFormatter() + formatter = JsonFormatter() logHandler.setFormatter(formatter) logger.addHandler(logHandler) ``` -### Customizing fields - -The fmt parser can also be overidden if you want to have required fields that differ from the default of just `message`. +### Output fields -These two invocations are equivalent: +You can control the logged fields by setting the `fmt` argument when creating the formatter. By default formatters will follow the same `style` of `fmt` as the `logging` module: `%`, `$`, and `{`. All [`LogRecord` attributes](https://docs.python.org/3/library/logging.html#logrecord-attributes) can be output using their name. ```python -class CustomJsonFormatter(jsonlogger.JsonFormatter): - def parse(self): - return self._fmt.split(';') - -formatter = CustomJsonFormatter('one;two') - -# is equivalent to: - -formatter = jsonlogger.JsonFormatter('%(one)s %(two)s') +formatter = JsonFormatter("{message}{asctime}{exc_info}", style="{") ``` You can also add extra fields to your json output by specifying a dict in place of message, as well as by specifying an `extra={}` argument. @@ -94,9 +86,9 @@ Contents of these dictionaries will be added at the root level of the entry and You can also use the `add_fields` method to add to or generally normalize the set of default set of fields, it is called for every log event. For example, to unify default fields with those provided by [structlog](http://www.structlog.org/) you could do something like this: ```python -class CustomJsonFormatter(jsonlogger.JsonFormatter): +class CustomJsonFormatter(JsonFormatter): def add_fields(self, log_record, record, message_dict): - super(CustomJsonFormatter, self).add_fields(log_record, record, message_dict) + super().add_fields(log_record, record, message_dict) if not log_record.get('timestamp'): # this doesn't use record.created, so it is slightly off now = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.%fZ') @@ -105,32 +97,54 @@ class CustomJsonFormatter(jsonlogger.JsonFormatter): log_record['level'] = log_record['level'].upper() else: log_record['level'] = record.levelname + return formatter = CustomJsonFormatter('%(timestamp)s %(level)s %(name)s %(message)s') ``` Items added to the log record will be included in *every* log message, no matter what the format requires. -### Adding custom object serialization +You can also override the `process_log_record` method to modify fields before they are serialized to JSON. + +```python +class SillyFormatter(JsonFormatter): + def process_log_record(log_record): + new_record = {k[::-1]: v for k, v in log_record.items()} + return new_record + +#### Supporting custom styles + +It is possible to support custom `style`s by setting `validate=False` and overriding the `parse` method. + +For example: + +```python +class CommaSupport(JsonFormatter): + def parse(self) -> list[str]: + if isinstance(self._style, str) and self._style == ",": + return self._fmt.split(",") + return super().parse() + +formatter = CommaSupport("message,asctime", style=",", validate=False) +``` + +### Custom object serialization + +Most formatters support `json_default` which is used to control how objects are serialized. For custom handling of object serialization you can specify default json object translator or provide a custom encoder ```python -def json_translate(obj): +def my_default(obj): if isinstance(obj, MyClass): return {"special": obj.special} -formatter = jsonlogger.JsonFormatter(json_default=json_translate, - json_encoder=json.JSONEncoder) -logHandler.setFormatter(formatter) - -logger.info({"special": "value", "run": 12}) -logger.info("classic message", extra={"special": "value", "run": 12}) +formatter = JsonFormatter(json_default=my_default) ``` ### Using a Config File -To use the module with a config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.jsonlogger.JsonFormatter`. Here is a sample config file. +To use the module with a config file using the [`fileConfig` function](https://docs.python.org/3/library/logging.config.html#logging.config.fileConfig), use the class `pythonjsonlogger.json.JsonFormatter`. Here is a sample config file. ```ini [loggers] @@ -180,7 +194,7 @@ Sample JSON with a full formatter (basically the log message from the unit test) "msecs": 506.24799728393555, "pathname": "tests/tests.py", "lineno": 60, - "asctime": ["12-05-05 22:11:08,506248"], + "asctime": "12-05-05 22:11:08,506248", "message": "testing logging format", "filename": "tests.py", "levelname": "INFO", diff --git a/pyproject.toml b/pyproject.toml index c2b1b8b..56f43e3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "python-json-logger" -version = "3.1.0.dev1" +version = "3.1.0.rc1" description = "JSON Log Formatter for the Python Logging Package" authors = [ {name = "Zakaria Zajac", email = "zak@madzak.com"}, diff --git a/src/pythonjsonlogger/core.py b/src/pythonjsonlogger/core.py index 722d00b..b38c27f 100644 --- a/src/pythonjsonlogger/core.py +++ b/src/pythonjsonlogger/core.py @@ -77,7 +77,7 @@ def str_to_object(obj: Any) -> Any: Args: obj: the object or string to process - *New in 4.0* + *New in 3.1* """ if not isinstance(obj, str): @@ -103,7 +103,7 @@ def merge_record_extra( rename_fields: an optional dict, used to rename field names in the output. e.g. Rename `levelname` to `log.level`: `{'levelname': 'log.level'}` - *Changed in 4.0*: `reserved` is now `Container[str]`. + *Changed in 3.1*: `reserved` is now `Container[str]`. """ if rename_fields is None: rename_fields = {} @@ -120,6 +120,8 @@ class BaseJsonFormatter(logging.Formatter): """Base class for pythonjsonlogger formatters Must not be used directly + + *New in 3.1* """ _style: Union[logging.PercentStyle, str] # type: ignore[assignment] @@ -144,9 +146,9 @@ def __init__( """ Args: fmt: string representing fields to log - datefmt: format to use when formatting asctime field + datefmt: format to use when formatting `asctime` field style: how to extract log fields from `fmt` - validate: validate fmt against style, if implementing a custom style you + validate: validate `fmt` against style, if implementing a custom `style` you must set this to `False`. defaults: ignored - kept for compatibility with python 3.10+ prefix: an optional string prefix added at the beginning of @@ -162,7 +164,7 @@ def __init__( to log record using string as key. If True boolean is passed, timestamp key will be "timestamp". Defaults to False/off. - *Changed in 4.0*: you can now use custom values for style by setting validate to `False`. + *Changed in 3.1*: you can now use custom values for style by setting validate to `False`. The value is stored in `self._style` as a string. The `parse` method will need to be overridden in order to support the new style. """ From 5cc6e3d73981d43b291d777d27ddb58acd8f0377 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Fri, 3 May 2024 20:17:46 +1000 Subject: [PATCH 24/27] Fix formatting --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f646c42..e849ec4 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,7 @@ class SillyFormatter(JsonFormatter): def process_log_record(log_record): new_record = {k[::-1]: v for k, v in log_record.items()} return new_record +``` #### Supporting custom styles From ce76b66bfc0519f77f21103d9e878f9ce884f2f8 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 5 May 2024 14:58:57 +1000 Subject: [PATCH 25/27] Add other encoders to README --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index e849ec4..75bca3f 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,13 @@ format = %(message)s class = pythonjsonlogger.jsonlogger.JsonFormatter ``` +### Alternate JSON Encoders + +The following JSON encoders are also supported: + +- [orjson](https://github.com/ijl/orjson) - `pythonjsonlogger.orjon.OrjsonFormatter` +- [msgspec](https://github.com/jcrist/msgspec) - `pythonjsonlogger.msgspec.MsgspecFormatter` + ## Example Output Sample JSON with a full formatter (basically the log message from the unit test). Every log message will appear on 1 line like a typical logger. From 44406d3ebb7189df39ef48b9c573109fece4cd86 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 5 May 2024 15:50:36 +1000 Subject: [PATCH 26/27] Update freezegun issue references --- tests/test_formatters.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index c80dd6f..9bd908b 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -334,10 +334,11 @@ def test_rename_reserved_attrs(env: LoggingEnvironment, class_: type[BaseJsonFor @pytest.mark.parametrize("class_", ALL_FORMATTERS) def test_default_encoder_with_timestamp(env: LoggingEnvironment, class_: type[BaseJsonFormatter]): if pythonjsonlogger.ORJSON_AVAILABLE and class_ is OrjsonFormatter: - # https://github.com/spulec/freezegun/issues/448#issuecomment-1686070438 + # https://github.com/ijl/orjson/issues/481 pytest.xfail() if pythonjsonlogger.MSGSPEC_AVAILABLE and class_ is MsgspecFormatter: + # https://github.com/jcrist/msgspec/issues/678 pytest.xfail() env.set_formatter(class_(timestamp=True)) From 5a6b959d50d1289106c1281efd773a584dbea088 Mon Sep 17 00:00:00 2001 From: Nicholas Hairs Date: Sun, 5 May 2024 15:54:09 +1000 Subject: [PATCH 27/27] Remove optional dependencies for specific encoders --- pyproject.toml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 56f43e3..b9e004f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,14 +43,6 @@ classifiers = [ GitHub = "https://github.com/nhairs/python-json-logger" [project.optional-dependencies] -orjson = [ - "orjson;implementation_name!='pypy'", -] - -msgspec = [ - "msgspec;implementation_name!='pypy'", -] - dev = [ ## Optional but required for dev "orjson;implementation_name!='pypy' and python_version<'3.13'",