From 7c454c74df846d79c10314a26a8aad3e06cd41ca Mon Sep 17 00:00:00 2001 From: kddubey Date: Mon, 1 Jan 2024 22:26:48 -0800 Subject: [PATCH 01/40] [WIP] Tap class from Pydantic model or dataclass --- demo_data_model.py | 68 +++++++++++++++ tap/__init__.py | 10 ++- tap/tap_class_from_data_model.py | 137 +++++++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 demo_data_model.py create mode 100644 tap/tap_class_from_data_model.py diff --git a/demo_data_model.py b/demo_data_model.py new file mode 100644 index 0000000..a8bbe4d --- /dev/null +++ b/demo_data_model.py @@ -0,0 +1,68 @@ +""" +Example: + +python demo_data_model.py \ +--arg_str test \ +--arg_list x y z \ +--arg_bool \ +-arg 2 +""" +from pydantic import BaseModel, Field +from tap import tapify, tap_class_from_data_model + + +class Model(BaseModel): + """ + My Pydantic Model which contains script args. + """ + + arg_str: str = Field(description="hello") + arg_bool: bool = Field(default=True, description=None) + arg_list: list[str] | None = Field(default=None, description="optional list") + + +def main(model: Model) -> None: + print("Parsed args into Model:") + print(model) + + +def to_number(string: str) -> float | int: + return float(string) if "." in string else int(string) + + +class ModelTap(tap_class_from_data_model(Model)): + # You can supply additional arguments here + argument_with_really_long_name: float | int = 3 + "This argument has a long name and will be aliased with a short one" + + def configure(self) -> None: + # You can still add special argument behavior + self.add_argument("-arg", "--argument_with_really_long_name", type=to_number) + + def process_args(self) -> None: + # You can still validate and modify arguments + # (You should do this in the Pydantic Model. I'm just demonstrating that this functionality is still possible) + if self.argument_with_really_long_name > 4: + raise ValueError("nope") + + # No auto-complete (and other niceties) for the super class attributes b/c this is a dynamic subclass. Sorry + if self.arg_bool: + self.arg_str += " processed" + + +if __name__ == "__main__": + # You don't have to subclass tap_class_from_data_model(Model) if you just want a plain argument parser: + # ModelTap = tap_class_from_data_model(Model) + args = ModelTap(description="Script description").parse_args() + print("Parsed args:") + print(args) + # Run the main function. Pydantic BaseModels ignore arguments which aren't one of their fields instead of raising an + # error + model = Model(**args.as_dict()) + main(model) + + +# This works but doesn't show the field description, and immediately returns a Model instance instead of a Tap class +# if __name__ == "__main__": +# model = tapify(Model) +# print(model) diff --git a/tap/__init__.py b/tap/__init__.py index b10e9d4..f45d6aa 100644 --- a/tap/__init__.py +++ b/tap/__init__.py @@ -2,5 +2,13 @@ from tap._version import __version__ from tap.tap import Tap from tap.tapify import tapify +from tap.tap_class_from_data_model import tap_class_from_data_model -__all__ = ["ArgumentError", "ArgumentTypeError", "Tap", "tapify", "__version__"] +__all__ = [ + "ArgumentError", + "ArgumentTypeError", + "Tap", + "tapify", + "tap_class_from_data_model", + "__version__", +] diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py new file mode 100644 index 0000000..17350df --- /dev/null +++ b/tap/tap_class_from_data_model.py @@ -0,0 +1,137 @@ +""" +Convert a data model to a Tap class. +""" + +import dataclasses +from typing import Any, Sequence + +import pydantic +from pydantic.fields import FieldInfo as PydanticFieldBaseModel +from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass + +from tap import Tap + + +_PydanticField = PydanticFieldBaseModel | PydanticFieldDataclass + + +@dataclasses.dataclass(frozen=True) +class _FieldData: + """ + Data about a field which is sufficient to inform a Tap variable/argument. + + TODO: maybe inject an inspect.Parameter instead + """ + + name: str + annotation: type + is_required: bool + default: Any + description: str | None = "" + + +def _field_data_from_dataclass(name: str, field: dataclasses.Field) -> _FieldData: + def is_required(field: dataclasses.Field) -> bool: + return field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING + + return _FieldData( + name, + field.type, + is_required(field), + field.default, + field.metadata.get("description"), + ) + + +def _field_data_from_pydantic(name: str, field: _PydanticField, annotation: type | None = None) -> _FieldData: + annotation = field.annotation if annotation is None else annotation + return _FieldData(name, annotation, field.is_required(), field.default, field.description) + + +def _fields_data(data_model: Any) -> list[_FieldData]: + if dataclasses.is_dataclass(data_model): + # This condition also holds for a Pydantic dataclass instance or model + name_to_field = {field.name: field for field in dataclasses.fields(data_model)} + elif isinstance(data_model, pydantic.BaseModel) or issubclass(data_model, pydantic.BaseModel): + # Check isinstance before issubclass. issubclass requires data_model is a class + name_to_field = data_model.model_fields + else: + raise TypeError( + "data_model must be a builtin or Pydantic dataclass (instance or class) or " + f"a Pydantic BaseModel (instance or class). Got {type(data_model)}" + ) + # TODO: instead of raising an error, inspect the signature of data_model to return a list[_FieldData] + # We could then refactor tapify to use _tap_class. It'd go like this: + # In tapify, add a kwarg for return_tap_class=False (then we don't need tap_class_from_data_model) + # fields_data = from inspecting the signature of the input class_or_function + # tap_class = _tap_class(fields_data) + # if return_tap_class: + # return tap_class + # tap = tap_class(description=description, explicit_bool=explicit_bool) + # command_line_args = tap.parse_args() + # command_line_args_dict = command_line_args.as_dict() + # return class_or_function(**command_line_args_dict) + + # It's possible to mix fields w/ classes, e.g., use pydantic Fields in a (builtin) dataclass, or use (builtin) + # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields + # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The + # solution is to iterate through the fields and check each type. + fields_data: list[_FieldData] = [] + for name, field in name_to_field.items(): + if isinstance(field, dataclasses.Field): + # Idiosyncrasy: if a pydantic Field is used in a pydantic dataclass, then field.default is a FieldInfo + # object instead of the field's default value. Furthermore, field.annotation is always NoneType. Luckily, + # the actual type of the field is stored in field.type + if isinstance(field.default, _PydanticField): + field_data = _field_data_from_pydantic(name, field.default, annotation=field.type) + else: + field_data = _field_data_from_dataclass(name, field) + elif isinstance(field, _PydanticField): + field_data = _field_data_from_pydantic(name, field) + else: + raise TypeError(f"Each field must be a dataclass or Pydantic field. Got {type(field)}") + fields_data.append(field_data) + return fields_data + + +def _tap_class(fields_data: Sequence[_FieldData]) -> type[Tap]: + class ArgParser(Tap): + # Overwriting configure would force a user to remember to call super().configure if they want to overwrite it + # Instead, overwrite _configure + def _configure(self): + # Add arguments from fields_data (extracted from a data model) + for field_data in fields_data: + variable = field_data.name + self._annotations[variable] = field_data.annotation + self.class_variables[variable] = {"comment": field_data.description or ""} + if field_data.is_required: + kwargs = {} + else: + kwargs = dict(required=False, default=field_data.default) + self.add_argument(f"--{variable}", **kwargs) + + super()._configure() + + return ArgParser + + +def tap_class_from_data_model(data_model: Any) -> type[Tap]: + """Convert a data model to a typed CLI argument parser. + + :param data_model: a builtin or Pydantic dataclass (class or instance) or Pydantic `BaseModel` (class or instance) + :return: a typed argument parser class + + Note + ---- + For a `data_model` containing builtin dataclass `field`s, argument descriptions are set to the `field`'s + `metadata["description"]`. + + For example:: + + from dataclasses import dataclass, field + + @dataclass class Data: + my_field: str = field(metadata={"description": "field description"}) + """ + fields_data = _fields_data(data_model) + return _tap_class(fields_data) From 420c1ff4ae7792d93cc1f5d2774aef474b332bea Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 2 Jan 2024 20:23:20 -0800 Subject: [PATCH 02/40] use str for Any, fix docstring --- tap/tap_class_from_data_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py index 17350df..308fef9 100644 --- a/tap/tap_class_from_data_model.py +++ b/tap/tap_class_from_data_model.py @@ -102,7 +102,7 @@ def _configure(self): # Add arguments from fields_data (extracted from a data model) for field_data in fields_data: variable = field_data.name - self._annotations[variable] = field_data.annotation + self._annotations[variable] = str if field_data.annotation is Any else field_data.annotation self.class_variables[variable] = {"comment": field_data.description or ""} if field_data.is_required: kwargs = {} @@ -130,7 +130,8 @@ def tap_class_from_data_model(data_model: Any) -> type[Tap]: from dataclasses import dataclass, field - @dataclass class Data: + @dataclass + class Data: my_field: str = field(metadata={"description": "field description"}) """ fields_data = _fields_data(data_model) From 9ace3f61fd0e9494fa75266aa656644a9d022532 Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 2 Jan 2024 21:01:39 -0800 Subject: [PATCH 03/40] Remove comments about refactor --- tap/tap_class_from_data_model.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py index 308fef9..b9e848b 100644 --- a/tap/tap_class_from_data_model.py +++ b/tap/tap_class_from_data_model.py @@ -60,18 +60,6 @@ def _fields_data(data_model: Any) -> list[_FieldData]: "data_model must be a builtin or Pydantic dataclass (instance or class) or " f"a Pydantic BaseModel (instance or class). Got {type(data_model)}" ) - # TODO: instead of raising an error, inspect the signature of data_model to return a list[_FieldData] - # We could then refactor tapify to use _tap_class. It'd go like this: - # In tapify, add a kwarg for return_tap_class=False (then we don't need tap_class_from_data_model) - # fields_data = from inspecting the signature of the input class_or_function - # tap_class = _tap_class(fields_data) - # if return_tap_class: - # return tap_class - # tap = tap_class(description=description, explicit_bool=explicit_bool) - # command_line_args = tap.parse_args() - # command_line_args_dict = command_line_args.as_dict() - # return class_or_function(**command_line_args_dict) - # It's possible to mix fields w/ classes, e.g., use pydantic Fields in a (builtin) dataclass, or use (builtin) # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The From ea6c0463c4da8c65bff127578edafcd89c60bfa9 Mon Sep 17 00:00:00 2001 From: Jesse Michel Date: Sat, 13 Jan 2024 13:06:45 -0500 Subject: [PATCH 04/40] add pydantic as a dependency so tests run --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c405119..106029e 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ license="MIT", packages=find_packages(), package_data={"tap": ["py.typed"]}, - install_requires=["typing-inspect >= 0.7.1", "docstring-parser >= 0.15"], + install_requires=["typing-inspect >= 0.7.1", "docstring-parser >= 0.15", "pydantic >= 2.5.0"], tests_require=["pytest"], python_requires=">=3.8", classifiers=[ From d44f48e58e1d0e84deef5a431d10d84253840fe1 Mon Sep 17 00:00:00 2001 From: Jesse Michel Date: Sat, 13 Jan 2024 13:12:56 -0500 Subject: [PATCH 05/40] use older verions --- tap/tap_class_from_data_model.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py index b9e848b..2c43da4 100644 --- a/tap/tap_class_from_data_model.py +++ b/tap/tap_class_from_data_model.py @@ -3,7 +3,7 @@ """ import dataclasses -from typing import Any, Sequence +from typing import Any, List, Optional, Sequence, Union import pydantic from pydantic.fields import FieldInfo as PydanticFieldBaseModel @@ -12,7 +12,7 @@ from tap import Tap -_PydanticField = PydanticFieldBaseModel | PydanticFieldDataclass +_PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] @dataclasses.dataclass(frozen=True) @@ -27,7 +27,7 @@ class _FieldData: annotation: type is_required: bool default: Any - description: str | None = "" + description: Optional[str] = "" def _field_data_from_dataclass(name: str, field: dataclasses.Field) -> _FieldData: @@ -43,12 +43,12 @@ def is_required(field: dataclasses.Field) -> bool: ) -def _field_data_from_pydantic(name: str, field: _PydanticField, annotation: type | None = None) -> _FieldData: +def _field_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[type] = None) -> _FieldData: annotation = field.annotation if annotation is None else annotation return _FieldData(name, annotation, field.is_required(), field.default, field.description) -def _fields_data(data_model: Any) -> list[_FieldData]: +def _fields_data(data_model: Any) -> List[_FieldData]: if dataclasses.is_dataclass(data_model): # This condition also holds for a Pydantic dataclass instance or model name_to_field = {field.name: field for field in dataclasses.fields(data_model)} @@ -64,7 +64,7 @@ def _fields_data(data_model: Any) -> list[_FieldData]: # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The # solution is to iterate through the fields and check each type. - fields_data: list[_FieldData] = [] + fields_data: List[_FieldData] = [] for name, field in name_to_field.items(): if isinstance(field, dataclasses.Field): # Idiosyncrasy: if a pydantic Field is used in a pydantic dataclass, then field.default is a FieldInfo @@ -82,7 +82,7 @@ def _fields_data(data_model: Any) -> list[_FieldData]: return fields_data -def _tap_class(fields_data: Sequence[_FieldData]) -> type[Tap]: +def _tap_class(fields_data: Sequence[_FieldData]) -> type: class ArgParser(Tap): # Overwriting configure would force a user to remember to call super().configure if they want to overwrite it # Instead, overwrite _configure From 2d92219700fb1ad232d9e3a2887a62ebb72f521b Mon Sep 17 00:00:00 2001 From: Jesse Michel Date: Sat, 13 Jan 2024 13:14:21 -0500 Subject: [PATCH 06/40] use typing type --- tap/tap_class_from_data_model.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py index 2c43da4..8ab079f 100644 --- a/tap/tap_class_from_data_model.py +++ b/tap/tap_class_from_data_model.py @@ -3,7 +3,7 @@ """ import dataclasses -from typing import Any, List, Optional, Sequence, Union +from typing import Any, List, Optional, Sequence, Type, Union import pydantic from pydantic.fields import FieldInfo as PydanticFieldBaseModel @@ -82,7 +82,7 @@ def _fields_data(data_model: Any) -> List[_FieldData]: return fields_data -def _tap_class(fields_data: Sequence[_FieldData]) -> type: +def _tap_class(fields_data: Sequence[_FieldData]) -> Type[Tap]: class ArgParser(Tap): # Overwriting configure would force a user to remember to call super().configure if they want to overwrite it # Instead, overwrite _configure @@ -103,7 +103,7 @@ def _configure(self): return ArgParser -def tap_class_from_data_model(data_model: Any) -> type[Tap]: +def tap_class_from_data_model(data_model: Any) -> Type[Tap]: """Convert a data model to a typed CLI argument parser. :param data_model: a builtin or Pydantic dataclass (class or instance) or Pydantic `BaseModel` (class or instance) From 603c6ad53666c9368521bcfa6242e1fc207c0e10 Mon Sep 17 00:00:00 2001 From: kddubey Date: Wed, 17 Jan 2024 20:58:18 -0800 Subject: [PATCH 07/40] convert_to_tap_class --- demo_data_model.py | 4 +- setup.py | 4 +- tap/__init__.py | 5 +- tap/tap_class_from_data_model.py | 126 ------------- tap/tapify.py | 307 +++++++++++++++++++++++++------ 5 files changed, 253 insertions(+), 193 deletions(-) delete mode 100644 tap/tap_class_from_data_model.py diff --git a/demo_data_model.py b/demo_data_model.py index a8bbe4d..c0c403c 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -8,7 +8,7 @@ -arg 2 """ from pydantic import BaseModel, Field -from tap import tapify, tap_class_from_data_model +from tap import tapify, convert_to_tap_class class Model(BaseModel): @@ -30,7 +30,7 @@ def to_number(string: str) -> float | int: return float(string) if "." in string else int(string) -class ModelTap(tap_class_from_data_model(Model)): +class ModelTap(convert_to_tap_class(Model)): # You can supply additional arguments here argument_with_really_long_name: float | int = 3 "This argument has a long name and will be aliased with a short one" diff --git a/setup.py b/setup.py index 106029e..e5f10f1 100644 --- a/setup.py +++ b/setup.py @@ -25,8 +25,8 @@ license="MIT", packages=find_packages(), package_data={"tap": ["py.typed"]}, - install_requires=["typing-inspect >= 0.7.1", "docstring-parser >= 0.15", "pydantic >= 2.5.0"], - tests_require=["pytest"], + install_requires=["typing-inspect >= 0.7.1", "docstring-parser >= 0.15"], + tests_require=["pytest", "pydantic >= 2.5.0"], python_requires=">=3.8", classifiers=[ "Programming Language :: Python :: 3", diff --git a/tap/__init__.py b/tap/__init__.py index f45d6aa..a5e4e74 100644 --- a/tap/__init__.py +++ b/tap/__init__.py @@ -1,14 +1,13 @@ from argparse import ArgumentError, ArgumentTypeError from tap._version import __version__ from tap.tap import Tap -from tap.tapify import tapify -from tap.tap_class_from_data_model import tap_class_from_data_model +from tap.tapify import tapify, convert_to_tap_class __all__ = [ "ArgumentError", "ArgumentTypeError", "Tap", "tapify", - "tap_class_from_data_model", + "convert_to_tap_class", "__version__", ] diff --git a/tap/tap_class_from_data_model.py b/tap/tap_class_from_data_model.py deleted file mode 100644 index 8ab079f..0000000 --- a/tap/tap_class_from_data_model.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Convert a data model to a Tap class. -""" - -import dataclasses -from typing import Any, List, Optional, Sequence, Type, Union - -import pydantic -from pydantic.fields import FieldInfo as PydanticFieldBaseModel -from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass - -from tap import Tap - - -_PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] - - -@dataclasses.dataclass(frozen=True) -class _FieldData: - """ - Data about a field which is sufficient to inform a Tap variable/argument. - - TODO: maybe inject an inspect.Parameter instead - """ - - name: str - annotation: type - is_required: bool - default: Any - description: Optional[str] = "" - - -def _field_data_from_dataclass(name: str, field: dataclasses.Field) -> _FieldData: - def is_required(field: dataclasses.Field) -> bool: - return field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING - - return _FieldData( - name, - field.type, - is_required(field), - field.default, - field.metadata.get("description"), - ) - - -def _field_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[type] = None) -> _FieldData: - annotation = field.annotation if annotation is None else annotation - return _FieldData(name, annotation, field.is_required(), field.default, field.description) - - -def _fields_data(data_model: Any) -> List[_FieldData]: - if dataclasses.is_dataclass(data_model): - # This condition also holds for a Pydantic dataclass instance or model - name_to_field = {field.name: field for field in dataclasses.fields(data_model)} - elif isinstance(data_model, pydantic.BaseModel) or issubclass(data_model, pydantic.BaseModel): - # Check isinstance before issubclass. issubclass requires data_model is a class - name_to_field = data_model.model_fields - else: - raise TypeError( - "data_model must be a builtin or Pydantic dataclass (instance or class) or " - f"a Pydantic BaseModel (instance or class). Got {type(data_model)}" - ) - # It's possible to mix fields w/ classes, e.g., use pydantic Fields in a (builtin) dataclass, or use (builtin) - # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields - # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The - # solution is to iterate through the fields and check each type. - fields_data: List[_FieldData] = [] - for name, field in name_to_field.items(): - if isinstance(field, dataclasses.Field): - # Idiosyncrasy: if a pydantic Field is used in a pydantic dataclass, then field.default is a FieldInfo - # object instead of the field's default value. Furthermore, field.annotation is always NoneType. Luckily, - # the actual type of the field is stored in field.type - if isinstance(field.default, _PydanticField): - field_data = _field_data_from_pydantic(name, field.default, annotation=field.type) - else: - field_data = _field_data_from_dataclass(name, field) - elif isinstance(field, _PydanticField): - field_data = _field_data_from_pydantic(name, field) - else: - raise TypeError(f"Each field must be a dataclass or Pydantic field. Got {type(field)}") - fields_data.append(field_data) - return fields_data - - -def _tap_class(fields_data: Sequence[_FieldData]) -> Type[Tap]: - class ArgParser(Tap): - # Overwriting configure would force a user to remember to call super().configure if they want to overwrite it - # Instead, overwrite _configure - def _configure(self): - # Add arguments from fields_data (extracted from a data model) - for field_data in fields_data: - variable = field_data.name - self._annotations[variable] = str if field_data.annotation is Any else field_data.annotation - self.class_variables[variable] = {"comment": field_data.description or ""} - if field_data.is_required: - kwargs = {} - else: - kwargs = dict(required=False, default=field_data.default) - self.add_argument(f"--{variable}", **kwargs) - - super()._configure() - - return ArgParser - - -def tap_class_from_data_model(data_model: Any) -> Type[Tap]: - """Convert a data model to a typed CLI argument parser. - - :param data_model: a builtin or Pydantic dataclass (class or instance) or Pydantic `BaseModel` (class or instance) - :return: a typed argument parser class - - Note - ---- - For a `data_model` containing builtin dataclass `field`s, argument descriptions are set to the `field`'s - `metadata["description"]`. - - For example:: - - from dataclasses import dataclass, field - - @dataclass - class Data: - my_field: str = field(metadata={"description": "field description"}) - """ - fields_data = _fields_data(data_model) - return _tap_class(fields_data) diff --git a/tap/tapify.py b/tap/tapify.py index 39e6cd7..419abbd 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -1,106 +1,293 @@ """Tapify module, which can initialize a class or run a function by parsing arguments from the command line.""" -from inspect import signature, Parameter -from typing import Any, Callable, List, Optional, Type, TypeVar, Union +import dataclasses +import inspect +from typing import Any, cast, Callable, List, Optional, Sequence, Type, TypeVar, Union -from docstring_parser import parse +from docstring_parser import Docstring, parse + +# TODO: don't require pydantic +import pydantic +from pydantic.fields import FieldInfo as PydanticFieldBaseModel +from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass from tap import Tap InputType = TypeVar("InputType") OutputType = TypeVar("OutputType") +_ClassOrFunction = Union[Callable[[InputType], OutputType], Type[OutputType]] -def tapify( - class_or_function: Union[Callable[[InputType], OutputType], Type[OutputType]], - known_only: bool = False, - command_line_args: Optional[List[str]] = None, - explicit_bool: bool = False, - **func_kwargs, -) -> OutputType: - """Tapify initializes a class or runs a function by parsing arguments from the command line. - :param class_or_function: The class or function to run with the provided arguments. - :param known_only: If true, ignores extra arguments and only parses known arguments. - :param command_line_args: A list of command line style arguments to parse (e.g., ['--arg', 'value']). - If None, arguments are parsed from the command line (default behavior). - :param explicit_bool: Booleans can be specified on the command line as "--arg True" or "--arg False" - rather than "--arg". Additionally, booleans can be specified by prefixes of True and False - with any capitalization as well as 1 or 0. - :param func_kwargs: Additional keyword arguments for the function. These act as default values when - parsing the command line arguments and overwrite the function defaults but - are overwritten by the parsed command line arguments. +@dataclasses.dataclass +class _ArgData: + """ + Data about an argument which is sufficient to inform a Tap variable/argument. """ - # Get signature from class or function - sig = signature(class_or_function) - # Parse class or function docstring in one line - if isinstance(class_or_function, type) and class_or_function.__init__.__doc__ is not None: - doc = class_or_function.__init__.__doc__ + name: str + + annotation: type + "The type of values this argument accepts" + + is_required: bool + "Whether or not the argument must be passed in" + + default: Any + "Value of the argument if the argument isn't passed in. This gets ignored if is_required" + + description: Optional[str] = "" + "Human-readable description of the argument" + + +@dataclasses.dataclass(frozen=True) +class _TapData: + """ + Data about a class' or function's arguments which are sufficient to inform a Tap class. + """ + + args_data: List[_ArgData] + "List of data about each argument in the class or function" + + has_kwargs: bool + "True if you can pass variable/extra kwargs to the class or function (as in **kwargs), else False" + + known_only: bool + "I don't know yet" + + +def _is_pydantic_base_model(obj: Any) -> bool: + # TODO: don't require pydantic + if inspect.isclass(obj): # issublcass requires that obj is a class + return issubclass(obj, pydantic.BaseModel) else: - doc = class_or_function.__doc__ + return isinstance(obj, pydantic.BaseModel) - # Parse docstring - docstring = parse(doc) - # Get the description of each argument in the class init or function - param_to_description = {param.arg_name: param.description for param in docstring.params} +def _tap_data_from_data_model( + data_model: Any, func_kwargs: dict[str, Any], param_to_description: dict[str, str] = None +) -> _TapData: + """ + Currently only works when `data_model` is a: + - builtin dataclass (class or instance) + - Pydantic dataclass (class or instance) + - Pydantic BaseModel (class or instance). - # Create a Tap object with a description from the docstring of the function or class - description = "\n".join(filter(None, (docstring.short_description, docstring.long_description))) - tap = Tap(description=description, explicit_bool=explicit_bool) + The advantage of this function over func:`_tap_data_from_class_or_function` is that descriptions are parsed, b/c we + look at the fields of the data model. + """ + param_to_description = param_to_description or {} + + def _arg_data_from_dataclass(name: str, field: dataclasses.Field) -> _ArgData: + def is_required(field: dataclasses.Field) -> bool: + return field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING + + description = param_to_description.get(name, field.metadata.get("description")) + return _ArgData( + name, + field.type, + is_required(field), + field.default, + description, + ) + + # TODO: don't require pydantic + _PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] + + def _arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[type] = None) -> _ArgData: + annotation = field.annotation if annotation is None else annotation + description = param_to_description.get(name, field.description) + return _ArgData(name, annotation, field.is_required(), field.default, description) + + if dataclasses.is_dataclass(data_model): + # This condition also holds for a Pydantic dataclass instance or model + name_to_field = {field.name: field for field in dataclasses.fields(data_model)} + has_kwargs = False + known_only = False # TODO: figure out what this was for dataclasses + elif _is_pydantic_base_model(data_model): + data_model: pydantic.BaseModel = cast(pydantic.BaseModel, data_model) + name_to_field = data_model.model_fields + is_extra_ok = data_model.model_config.get("extra", "ignore") != "forbid" + has_kwargs = is_extra_ok + known_only = not is_extra_ok # TODO: figure out what this means + else: + raise TypeError( + "data_model must be a builtin or Pydantic dataclass (instance or class) or " + f"a Pydantic BaseModel (instance or class). Got {type(data_model)}" + ) + # It's possible to mix fields w/ classes, e.g., use pydantic Fields in a (builtin) dataclass, or use (builtin) + # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields + # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The + # solution is to iterate through the fields and check each type. + args_data: List[_ArgData] = [] + for name, field in name_to_field.items(): + if isinstance(field, dataclasses.Field): + # Idiosyncrasy: if a pydantic Field is used in a pydantic dataclass, then field.default is a FieldInfo + # object instead of the field's default value. Furthermore, field.annotation is always NoneType. Luckily, + # the actual type of the field is stored in field.type + if isinstance(field.default, _PydanticField): + arg_data = _arg_data_from_pydantic(name, field.default, annotation=field.type) + else: + arg_data = _arg_data_from_dataclass(name, field) + elif isinstance(field, _PydanticField): + arg_data = _arg_data_from_pydantic(name, field) + else: + raise TypeError(f"Each field must be a dataclass or Pydantic field. Got {type(field)}") + # Handle case where func_kwargs is supplied + if name in func_kwargs: + arg_data.default = func_kwargs[name] + arg_data.is_required = False + del func_kwargs[name] + args_data.append(arg_data) + return _TapData(args_data, has_kwargs, known_only) - # Keep track of whether **kwargs was provided + +def _tap_data_from_class_or_function( + class_or_function: _ClassOrFunction, func_kwargs: dict[str, Any], param_to_description: dict[str, str] +) -> _TapData: + """ + Note + ---- + Modifies `func_kwargs` + """ + args_data: List[_ArgData] = [] has_kwargs = False + known_only = False - # Add arguments of class init or function to the Tap object - for param_name, param in sig.parameters.items(): - tap_kwargs = {} + sig = inspect.signature(class_or_function) + for param_name, param in sig.parameters.items(): # Skip **kwargs - if param.kind == Parameter.VAR_KEYWORD: + if param.kind == inspect.Parameter.VAR_KEYWORD: has_kwargs = True known_only = True continue - # Get type of the argument - if param.annotation != Parameter.empty: - # Any type defaults to str (needed for dataclasses where all non-default attributes must have a type) - if param.annotation is Any: - tap._annotations[param.name] = str - # Otherwise, get the type of the argument - else: - tap._annotations[param.name] = param.annotation + if param.annotation != inspect.Parameter.empty: + annotation = param.annotation + else: + annotation = Any - # Get the default or required of the argument if param.name in func_kwargs: - tap_kwargs["default"] = func_kwargs[param.name] + is_required = False + default = func_kwargs[param.name] del func_kwargs[param.name] - elif param.default != Parameter.empty: - tap_kwargs["default"] = param.default + elif param.default != inspect.Parameter.empty: + is_required = False + default = param.default else: - tap_kwargs["required"] = True + is_required = True + default = inspect.Parameter.empty # Can be set to anything. It'll be ignored + + arg_data = _ArgData( + name=param_name, + annotation=annotation, + is_required=is_required, + default=default, + description=param_to_description.get(param.name), + ) + args_data.append(arg_data) + return _TapData(args_data, has_kwargs, known_only) + + +def _is_data_model(obj: Any) -> bool: + return dataclasses.is_dataclass(obj) or _is_pydantic_base_model(obj) + + +def _docstring(class_or_function): + """Parse class or function docstring in one line""" + if inspect.isclass(class_or_function) and class_or_function.__init__.__doc__ is not None: + doc = class_or_function.__init__.__doc__ + else: + doc = class_or_function.__doc__ + return parse(doc) + + +def _tap_data(class_or_function: _ClassOrFunction, docstring: Docstring, func_kwargs): + # Get the description of each argument in the class init or function + param_to_description = {param.arg_name: param.description for param in docstring.params} + if _is_data_model(class_or_function): + # TODO: allow passing func_kwargs to a Pydantic BaseModel + return _tap_data_from_data_model(class_or_function, func_kwargs, param_to_description) + else: + return _tap_data_from_class_or_function(class_or_function, func_kwargs, param_to_description) + + +def _tap_class(args_data: Sequence[_ArgData]) -> Type[Tap]: + class ArgParser(Tap): + # Overwriting configure would force a user to remember to call super().configure if they want to overwrite it + # Instead, overwrite _configure + def _configure(self): + for arg_data in args_data: + variable = arg_data.name + self._annotations[variable] = str if arg_data.annotation is Any else arg_data.annotation + self.class_variables[variable] = {"comment": arg_data.description or ""} + if arg_data.is_required: + kwargs = {} + else: + kwargs = dict(required=False, default=arg_data.default) + self.add_argument(f"--{variable}", **kwargs) + + super()._configure() + + return ArgParser + + +def convert_to_tap_class(class_or_function: _ClassOrFunction, **func_kwargs) -> Type[Tap]: + """Creates a `Tap` class from `class_or_function`. This can be subclassed to add custom argument handling and + instantiated to create a typed argument parser. + + :param class_or_function: The class or function to run with the provided arguments. + :param func_kwargs: Additional keyword arguments for the function. These act as default values when + parsing the command line arguments and overwrite the function defaults but + are overwritten by the parsed command line arguments. + """ + docstring = _docstring(class_or_function) + tap_data = _tap_data(class_or_function, docstring, func_kwargs) + return _tap_class(tap_data.args_data) - # Get the help string of the argument - if param.name in param_to_description: - tap.class_variables[param.name] = {"comment": param_to_description[param.name]} - # Add the argument to the Tap object - tap._add_argument(f"--{param_name}", **tap_kwargs) +def tapify( + class_or_function: _ClassOrFunction, + known_only: bool = False, + command_line_args: Optional[List[str]] = None, + explicit_bool: bool = False, + **func_kwargs, +) -> OutputType: + """Tapify initializes a class or runs a function by parsing arguments from the command line. + + :param class_or_function: The class or function to run with the provided arguments. + :param known_only: If true, ignores extra arguments and only parses known arguments. + :param command_line_args: A list of command line style arguments to parse (e.g., ['--arg', 'value']). + If None, arguments are parsed from the command line (default behavior). + :param explicit_bool: Booleans can be specified on the command line as "--arg True" or "--arg False" + rather than "--arg". Additionally, booleans can be specified by prefixes of True and False + with any capitalization as well as 1 or 0. + :param func_kwargs: Additional keyword arguments for the function. These act as default values when + parsing the command line arguments and overwrite the function defaults but + are overwritten by the parsed command line arguments. + """ + # We don't directly call convert_to_tap_class b/c we need tap_data, not just tap_class + docstring = _docstring(class_or_function) + tap_data = _tap_data(class_or_function, docstring, func_kwargs) + tap_class = _tap_class(tap_data.args_data) + # Create a Tap object with a description from the docstring of the class or function + description = "\n".join(filter(None, (docstring.short_description, docstring.long_description))) + tap = tap_class(description=description, explicit_bool=explicit_bool) # If any func_kwargs remain, they are not used in the function, so raise an error + known_only = known_only or tap_data.known_only if func_kwargs and not known_only: raise ValueError(f"Unknown keyword arguments: {func_kwargs}") # Parse command line arguments - command_line_args = tap.parse_args(args=command_line_args, known_only=known_only) + command_line_args: Tap = tap.parse_args(args=command_line_args, known_only=known_only) # Get command line arguments as a dictionary command_line_args_dict = command_line_args.as_dict() # Get **kwargs from extra command line arguments - if has_kwargs: + if tap_data.has_kwargs: kwargs = {tap.extra_args[i].lstrip("-"): tap.extra_args[i + 1] for i in range(0, len(tap.extra_args), 2)} - command_line_args_dict.update(kwargs) # Initialize the class or run the function with the parsed arguments From 27df9038e5e6bb8e0b4a19684c0c8e1f3fef519d Mon Sep 17 00:00:00 2001 From: kddubey Date: Wed, 17 Jan 2024 21:07:43 -0800 Subject: [PATCH 08/40] add dev extra --- .github/workflows/tests.yml | 2 +- setup.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3470872..b436e0c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,7 +36,7 @@ jobs: git config --global user.name "Your Name" python -m pip install --upgrade pip python -m pip install flake8 pytest - python -m pip install -e . + python -m pip install -e ".[dev]" - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/setup.py b/setup.py index e5f10f1..f4d3b3c 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,11 @@ with open("README.md", encoding="utf-8") as f: long_description = f.read() +test_requirements = [ + "pydantic >= 2.5.0", + "pytest", +] + setup( name="typed-argument-parser", version=__version__, @@ -26,7 +31,8 @@ packages=find_packages(), package_data={"tap": ["py.typed"]}, install_requires=["typing-inspect >= 0.7.1", "docstring-parser >= 0.15"], - tests_require=["pytest", "pydantic >= 2.5.0"], + tests_require=test_requirements, + extras_require={"dev": test_requirements}, python_requires=">=3.8", classifiers=[ "Programming Language :: Python :: 3", From 89123b755b72ebefc826ce761bea976572ded5ef Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 14:35:51 -0800 Subject: [PATCH 09/40] fix desc for known_only --- tap/tapify.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index 419abbd..3c90e07 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -52,7 +52,7 @@ class _TapData: "True if you can pass variable/extra kwargs to the class or function (as in **kwargs), else False" known_only: bool - "I don't know yet" + "If true, ignore extra arguments and only parse known arguments" def _is_pydantic_base_model(obj: Any) -> bool: @@ -102,13 +102,14 @@ def _arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Option # This condition also holds for a Pydantic dataclass instance or model name_to_field = {field.name: field for field in dataclasses.fields(data_model)} has_kwargs = False - known_only = False # TODO: figure out what this was for dataclasses + known_only = False elif _is_pydantic_base_model(data_model): data_model: pydantic.BaseModel = cast(pydantic.BaseModel, data_model) name_to_field = data_model.model_fields + # TODO: understand current behavior of extra arg handling for Pydantic models through tapify is_extra_ok = data_model.model_config.get("extra", "ignore") != "forbid" has_kwargs = is_extra_ok - known_only = not is_extra_ok # TODO: figure out what this means + known_only = is_extra_ok else: raise TypeError( "data_model must be a builtin or Pydantic dataclass (instance or class) or " From 0b4f202bc9c41f920570648b37c30a6dd4a8d9b8 Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 16:10:12 -0800 Subject: [PATCH 10/40] dont require pydantic --- tap/tapify.py | 59 ++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 26 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index 3c90e07..8863fa0 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -1,14 +1,22 @@ """Tapify module, which can initialize a class or run a function by parsing arguments from the command line.""" import dataclasses import inspect -from typing import Any, cast, Callable, List, Optional, Sequence, Type, TypeVar, Union +from typing import Any, Callable, List, Optional, Sequence, Type, TypeVar, Union from docstring_parser import Docstring, parse -# TODO: don't require pydantic -import pydantic -from pydantic.fields import FieldInfo as PydanticFieldBaseModel -from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass +try: + import pydantic +except ModuleNotFoundError: + BaseModel = type("BaseModel", (object,), {}) + _PydanticField = type("_PydanticField", (object,), {}) +else: + from pydantic import BaseModel + from pydantic.fields import FieldInfo as PydanticFieldBaseModel + from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass + + _PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] + from tap import Tap @@ -26,7 +34,7 @@ class _ArgData: name: str - annotation: type + annotation: Type "The type of values this argument accepts" is_required: bool @@ -56,11 +64,10 @@ class _TapData: def _is_pydantic_base_model(obj: Any) -> bool: - # TODO: don't require pydantic if inspect.isclass(obj): # issublcass requires that obj is a class - return issubclass(obj, pydantic.BaseModel) + return issubclass(obj, BaseModel) else: - return isinstance(obj, pydantic.BaseModel) + return isinstance(obj, BaseModel) def _tap_data_from_data_model( @@ -72,12 +79,16 @@ def _tap_data_from_data_model( - Pydantic dataclass (class or instance) - Pydantic BaseModel (class or instance). - The advantage of this function over func:`_tap_data_from_class_or_function` is that descriptions are parsed, b/c we - look at the fields of the data model. + The advantage of this function over func:`_tap_data_from_class_or_function` is that descriptions are parsed, b/c + this function look at the fields of the data model. + + Note + ---- + Deletes redundant keys from `func_kwargs` """ param_to_description = param_to_description or {} - def _arg_data_from_dataclass(name: str, field: dataclasses.Field) -> _ArgData: + def arg_data_from_dataclass(name: str, field: dataclasses.Field) -> _ArgData: def is_required(field: dataclasses.Field) -> bool: return field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING @@ -90,21 +101,17 @@ def is_required(field: dataclasses.Field) -> bool: description, ) - # TODO: don't require pydantic - _PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] - - def _arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[type] = None) -> _ArgData: + def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[Type] = None) -> _ArgData: annotation = field.annotation if annotation is None else annotation description = param_to_description.get(name, field.description) return _ArgData(name, annotation, field.is_required(), field.default, description) + # Determine what type of data model it is and extract fields accordingly if dataclasses.is_dataclass(data_model): - # This condition also holds for a Pydantic dataclass instance or model name_to_field = {field.name: field for field in dataclasses.fields(data_model)} has_kwargs = False known_only = False elif _is_pydantic_base_model(data_model): - data_model: pydantic.BaseModel = cast(pydantic.BaseModel, data_model) name_to_field = data_model.model_fields # TODO: understand current behavior of extra arg handling for Pydantic models through tapify is_extra_ok = data_model.model_config.get("extra", "ignore") != "forbid" @@ -115,6 +122,7 @@ def _arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Option "data_model must be a builtin or Pydantic dataclass (instance or class) or " f"a Pydantic BaseModel (instance or class). Got {type(data_model)}" ) + # It's possible to mix fields w/ classes, e.g., use pydantic Fields in a (builtin) dataclass, or use (builtin) # dataclass fields in a pydantic BaseModel. It's also possible to use (builtin) dataclass fields and pydantic Fields # in the same data model. Therefore, the type of the data model doesn't determine the type of each field. The @@ -126,11 +134,11 @@ def _arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Option # object instead of the field's default value. Furthermore, field.annotation is always NoneType. Luckily, # the actual type of the field is stored in field.type if isinstance(field.default, _PydanticField): - arg_data = _arg_data_from_pydantic(name, field.default, annotation=field.type) + arg_data = arg_data_from_pydantic(name, field.default, annotation=field.type) else: - arg_data = _arg_data_from_dataclass(name, field) + arg_data = arg_data_from_dataclass(name, field) elif isinstance(field, _PydanticField): - arg_data = _arg_data_from_pydantic(name, field) + arg_data = arg_data_from_pydantic(name, field) else: raise TypeError(f"Each field must be a dataclass or Pydantic field. Got {type(field)}") # Handle case where func_kwargs is supplied @@ -148,7 +156,7 @@ def _tap_data_from_class_or_function( """ Note ---- - Modifies `func_kwargs` + Deletes redundant keys from `func_kwargs` """ args_data: List[_ArgData] = [] has_kwargs = False @@ -194,7 +202,7 @@ def _is_data_model(obj: Any) -> bool: return dataclasses.is_dataclass(obj) or _is_pydantic_base_model(obj) -def _docstring(class_or_function): +def _docstring(class_or_function) -> Docstring: """Parse class or function docstring in one line""" if inspect.isclass(class_or_function) and class_or_function.__init__.__doc__ is not None: doc = class_or_function.__init__.__doc__ @@ -203,8 +211,7 @@ def _docstring(class_or_function): return parse(doc) -def _tap_data(class_or_function: _ClassOrFunction, docstring: Docstring, func_kwargs): - # Get the description of each argument in the class init or function +def _tap_data(class_or_function: _ClassOrFunction, docstring: Docstring, func_kwargs) -> _TapData: param_to_description = {param.arg_name: param.description for param in docstring.params} if _is_data_model(class_or_function): # TODO: allow passing func_kwargs to a Pydantic BaseModel @@ -248,7 +255,7 @@ def convert_to_tap_class(class_or_function: _ClassOrFunction, **func_kwargs) -> def tapify( - class_or_function: _ClassOrFunction, + class_or_function: Union[Callable[[InputType], OutputType], Type[OutputType]], known_only: bool = False, command_line_args: Optional[List[str]] = None, explicit_bool: bool = False, From a11a0881b35d91d7fbab0dad6b4edb5c25ee5d93 Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 17:32:14 -0800 Subject: [PATCH 11/40] fix docstring extraction --- tap/tapify.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index 8863fa0..c758990 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -204,10 +204,11 @@ def _is_data_model(obj: Any) -> bool: def _docstring(class_or_function) -> Docstring: """Parse class or function docstring in one line""" - if inspect.isclass(class_or_function) and class_or_function.__init__.__doc__ is not None: - doc = class_or_function.__init__.__doc__ - else: + is_function = not inspect.isclass(class_or_function) + if is_function or _is_pydantic_base_model(class_or_function): doc = class_or_function.__doc__ + else: + doc = class_or_function.__init__.__doc__ or class_or_function.__doc__ return parse(doc) From c0d226f59fcf25e9e040d554d0365fe639ef403a Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 17:33:29 -0800 Subject: [PATCH 12/40] test pydantic BaseModel --- tests/test_tapify.py | 229 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 210 insertions(+), 19 deletions(-) diff --git a/tests/test_tapify.py b/tests/test_tapify.py index d121c6d..3793eb3 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -1,11 +1,13 @@ import contextlib -from dataclasses import dataclass +from dataclasses import dataclass, field import io import sys from typing import Dict, List, Optional, Tuple, Any import unittest from unittest import TestCase +import pydantic + from tap import tapify @@ -49,7 +51,11 @@ class PieDataclass: def __eq__(self, other: float) -> bool: return other == pie() - for class_or_function in [pie, Pie, PieDataclass]: + class PieModel(pydantic.BaseModel): + def __eq__(self, other: float) -> bool: + return other == pie() + + for class_or_function in [pie, Pie, PieDataclass, PieModel]: self.assertEqual(tapify(class_or_function, command_line_args=[]), 3.14) def test_tapify_simple_types(self): @@ -74,7 +80,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + a: int + simple: str + test: float + of: float + types: bool + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--of", "2.718", "--types"], @@ -107,7 +123,18 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + a: int + simple: str + test: float + of: float = -0.3 + types: bool = pydantic.Field(False) + wow: str = pydantic.dataclasses.Field("abc") # mixing field types should be ok + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--types", "--wow", "wee"], @@ -135,7 +162,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + + complexity: List[str] + requires: Tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -176,10 +213,30 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + + complexity: list[int] + requires: tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, - command_line_args=["--complexity", "1", "2", "3", "--requires", "1", "0", "--intelligence", "jesse",], + command_line_args=[ + "--complexity", + "1", + "2", + "3", + "--requires", + "1", + "0", + "--intelligence", + "jesse", + ], ) self.assertEqual(output, "1 2 3 1 0 Person(jesse)") @@ -225,7 +282,19 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + + complexity: List[str] + requires: Tuple[int, int] = (2, 5) + intelligence: Person = Person("kyle") + maybe: Optional[str] = None + possibly: Optional[str] = None + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -263,7 +332,15 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.many, self.args) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + so: int + many: float + args: str + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.many, self.args) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--many", "9.3"]) @@ -286,7 +363,16 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments + + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"]) @@ -309,7 +395,14 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"], known_only=True ) @@ -339,10 +432,28 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, - command_line_args=["--i", "23", "--args", "wow", "--like", "3.03",], + command_line_args=[ + "--i", + "23", + "--args", + "wow", + "--like", + "3.03", + ], known_only=True, w="hello", k=5, @@ -375,11 +486,31 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments + + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: with self.assertRaises(ValueError): tapify( class_or_function, - command_line_args=["--i", "23", "--args", "wow", "--like", "3.03",], + command_line_args=[ + "--i", + "23", + "--args", + "wow", + "--like", + "3.03", + ], w="hello", k=5, like=3.4, @@ -404,7 +535,15 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.problems) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Problems + + problems: Problems + + def __eq__(self, other: str) -> bool: + return other == concat(self.problems) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify(class_or_function, command_line_args=[], problems=Problems("oh", "no!")) self.assertEqual(output, "Problems(oh, no!)") @@ -448,7 +587,20 @@ def __eq__(self, other: str) -> bool: self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 ) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + untyped_1: Any + typed_1: int + untyped_2: Any = 5 + typed_2: str = "now" + untyped_3: Any = "hi" + typed_3: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat( + self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 + ) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -491,7 +643,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModel(pydantic.BaseModel): + """Concatenate three numbers.""" + + a: int + b: int + c: int + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: output_1 = tapify(class_or_function, command_line_args=["--a", "1", "--b", "2", "--c", "3"]) output_2 = tapify(class_or_function, command_line_args=["--a", "4", "--b", "5", "--c", "6"]) @@ -555,7 +717,36 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - for class_or_function in [concat, Concat, ConcatDataclass]: + class ConcatModelDocstring(pydantic.BaseModel): + """Concatenate three numbers. + + :param a: The first number. + :param b: The second number. + :param c: The third number. + """ + + a: int + b: int + c: int + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + + class ConcatModelFields(pydantic.BaseModel): + """Concatenate three numbers. + + :param a: The first number. + """ + + # Mixing field types should be ok + a: int + b: int = pydantic.dataclasses.Field(description="The second number.") + c: int = field(metadata={"description": "The third number."}) + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + + for class_or_function in [concat, Concat, ConcatDataclass, ConcatModelDocstring, ConcatModelFields]: f = io.StringIO() with contextlib.redirect_stdout(f): with self.assertRaises(SystemExit): From 0f4df975f31b9d47df9906e2c2addffe8e3f42b8 Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 18:43:57 -0800 Subject: [PATCH 13/40] test pydantic dataclass --- tests/test_tapify.py | 207 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 192 insertions(+), 15 deletions(-) diff --git a/tests/test_tapify.py b/tests/test_tapify.py index 3793eb3..dbd8a03 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -51,11 +51,16 @@ class PieDataclass: def __eq__(self, other: float) -> bool: return other == pie() + @pydantic.dataclasses.dataclass + class PieDataclassPydantic: + def __eq__(self, other: float) -> bool: + return other == pie() + class PieModel(pydantic.BaseModel): def __eq__(self, other: float) -> bool: return other == pie() - for class_or_function in [pie, Pie, PieDataclass, PieModel]: + for class_or_function in [pie, Pie, PieDataclass, PieDataclassPydantic, PieModel]: self.assertEqual(tapify(class_or_function, command_line_args=[]), 3.14) def test_tapify_simple_types(self): @@ -80,6 +85,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + a: int + simple: str + test: float + of: float + types: bool + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types) + class ConcatModel(pydantic.BaseModel): a: int simple: str @@ -90,7 +106,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--of", "2.718", "--types"], @@ -123,6 +139,18 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + a: int + simple: str + test: float + of: float = -0.3 + types: bool = pydantic.dataclasses.Field(False) + wow: str = pydantic.Field("abc") # mixing field types should be ok + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) + class ConcatModel(pydantic.BaseModel): a: int simple: str @@ -134,7 +162,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--types", "--wow", "wee"], @@ -162,6 +190,15 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: List[str] + requires: Tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person @@ -172,7 +209,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -213,6 +250,15 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: list[int] + requires: tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person @@ -223,7 +269,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -282,6 +328,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: List[str] + requires: Tuple[int, int] = (2, 5) + intelligence: Person = Person("kyle") + maybe: Optional[str] = None + possibly: Optional[str] = None + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person @@ -294,7 +351,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -332,6 +389,15 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.many, self.args) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + many: float + args: str + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.many, self.args) + class ConcatModel(pydantic.BaseModel): so: int many: float @@ -340,7 +406,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.so, self.many, self.args) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--many", "9.3"]) @@ -363,6 +429,14 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments @@ -372,7 +446,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"]) @@ -395,6 +469,14 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) + class ConcatModel(pydantic.BaseModel): so: int few: float @@ -402,7 +484,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"], known_only=True ) @@ -432,6 +514,18 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + class ConcatModel(pydantic.BaseModel): i: int like: float @@ -443,7 +537,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -486,6 +580,18 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments @@ -499,7 +605,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: with self.assertRaises(ValueError): tapify( class_or_function, @@ -535,6 +641,13 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.problems) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) + class ConcatDataclassPydantic: + problems: Problems + + def __eq__(self, other: str) -> bool: + return other == concat(self.problems) + class ConcatModel(pydantic.BaseModel): model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Problems @@ -543,7 +656,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.problems) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify(class_or_function, command_line_args=[], problems=Problems("oh", "no!")) self.assertEqual(output, "Problems(oh, no!)") @@ -587,6 +700,20 @@ def __eq__(self, other: str) -> bool: self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 ) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + untyped_1: Any + typed_1: int + untyped_2: Any = 5 + typed_2: str = "now" + untyped_3: Any = "hi" + typed_3: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat( + self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 + ) + class ConcatModel(pydantic.BaseModel): untyped_1: Any typed_1: int @@ -600,7 +727,7 @@ def __eq__(self, other: str) -> bool: self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 ) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output = tapify( class_or_function, command_line_args=[ @@ -643,6 +770,17 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + """Concatenate three numbers.""" + + a: int + b: int + c: int + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + class ConcatModel(pydantic.BaseModel): """Concatenate three numbers.""" @@ -653,7 +791,7 @@ class ConcatModel(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModel]: + for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: output_1 = tapify(class_or_function, command_line_args=["--a", "1", "--b", "2", "--c", "3"]) output_2 = tapify(class_or_function, command_line_args=["--a", "4", "--b", "5", "--c", "6"]) @@ -717,6 +855,37 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydanticDocstring: + """Concatenate three numbers. + + :param a: The first number. + :param b: The second number. + :param c: The third number. + """ + + a: int + b: int + c: int + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + + @pydantic.dataclasses.dataclass + class ConcatDataclassPydanticFields: + """Concatenate three numbers. + + :param a: The first number. + """ + + # Mixing field types should be ok + a: int + b: int = pydantic.Field(description="The second number.") + c: int = pydantic.dataclasses.Field(description="The third number.") + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) + class ConcatModelDocstring(pydantic.BaseModel): """Concatenate three numbers. @@ -746,7 +915,15 @@ class ConcatModelFields(pydantic.BaseModel): def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatModelDocstring, ConcatModelFields]: + for class_or_function in [ + concat, + Concat, + ConcatDataclass, + ConcatDataclassPydanticDocstring, + ConcatDataclassPydanticFields, + ConcatModelDocstring, + ConcatModelFields, + ]: f = io.StringIO() with contextlib.redirect_stdout(f): with self.assertRaises(SystemExit): From 53eb20325017ab0d8be0b703a12ad9df1db820e9 Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 18 Jan 2024 23:57:07 -0800 Subject: [PATCH 14/40] basic test convert_to_tap_class --- tests/test_convert_to_tap_class.py | 175 +++++++++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 tests/test_convert_to_tap_class.py diff --git a/tests/test_convert_to_tap_class.py b/tests/test_convert_to_tap_class.py new file mode 100644 index 0000000..f096bcf --- /dev/null +++ b/tests/test_convert_to_tap_class.py @@ -0,0 +1,175 @@ +""" +Tests `tap.convert_to_tap_class`. + +TODO: I might redesign this soon. It's not thorough yet. +""" +import dataclasses +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union + +import pydantic +import pytest + +from tap import convert_to_tap_class, Tap + + +@dataclasses.dataclass +class _Args: + """These are the argument names which every type of class or function must contain.""" + + arg_str: str = dataclasses.field(metadata=dict(description="some string")) + arg_bool: bool = True + arg_list: Optional[List[str]] = dataclasses.field(default=None, metadata=dict(description="some list of strings")) + + +def _monkeypatch_eq(cls): + """Monkey-patches `cls.__eq__` to check that the attribute values are equal to a dataclass representation of them""" + + def _equality(self, other: _Args) -> bool: + return _Args(self.arg_str, self.arg_bool, self.arg_list) == other + + cls.__eq__ = _equality + return cls + + +# Define a few different classes or functions which all take the same arguments (same by name, annotation, and default +# if not required) + + +def function(arg_str: str, arg_bool: bool = True, arg_list: Optional[List[str]] = None) -> _Args: + """ + :param arg_str: some string + :param arg_list: some list of strings + """ + return _Args(*locals().values()) + + +@_monkeypatch_eq +class Class: + def __init__(self, arg_str: str, arg_bool: bool = True, arg_list: Optional[List[str]] = None): + """ + :param arg_str: some string + :param arg_list: some list of strings + """ + self.arg_str = arg_str + self.arg_bool = arg_bool + self.arg_list = arg_list + + +DataclassBuiltin = _Args +"""Dataclass (builtin)""" + + +@_monkeypatch_eq +@pydantic.dataclasses.dataclass +class DataclassPydantic: + """Dataclass (pydantic)""" + + arg_str: str = pydantic.dataclasses.Field(description="some string") + arg_bool: bool = pydantic.dataclasses.Field(default=True) + arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") + + +@_monkeypatch_eq +class Model(pydantic.BaseModel): + """Pydantic model""" + + arg_str: str = pydantic.Field(description="some string") + arg_bool: bool = pydantic.Field(default=True) + arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") + + +# Define some functions which take a class or function and calls `tap.convert_to_tap_class` on it to create a `tap.Tap` +# subclass (class, not instance). Call this type of function a subclasser + + +def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: + return convert_to_tap_class(class_or_function) # plain subclass / do nothing + + +# TODO: use this. Will need to change how the test is parametrized b/c the output will depend on using +# subclass_tap_simple vs subclass_tap_weird +def subclass_tap_weird(class_or_function): + def to_number(string: str) -> float | int: + return float(string) if "." in string else int(string) + + class TapSubclass(convert_to_tap_class(class_or_function)): + # You can supply additional arguments here + argument_with_really_long_name: float | int = 3 + "This argument has a long name and will be aliased with a short one" + + def configure(self) -> None: + # You can still add special argument behavior + self.add_argument("-arg", "--argument_with_really_long_name", type=to_number) + + def process_args(self) -> None: + # You can still validate and modify arguments + if self.argument_with_really_long_name > 4: + raise ValueError("nope") + + # No auto-complete (and other niceties) for the super class attributes b/c this is a dynamic subclass. Sorry + if self.arg_bool: + self.arg_str += " processed" + + return TapSubclass + + +@pytest.mark.parametrize("subclass_tap", [subclass_tap_simple]) +@pytest.mark.parametrize( + "class_or_function", + [ + function, + Class, + DataclassBuiltin, + DataclassBuiltin( + "convert_to_tap_class works on instances of data models (for free). It ignores the attribute values", + arg_bool=False, + arg_list=["doesn't", "matter"], + ), + DataclassPydantic, + DataclassPydantic(arg_str="...", arg_bool=False, arg_list=[]), + Model, + Model(arg_str="...", arg_bool=False, arg_list=[]), + ], +) +@pytest.mark.parametrize( + "args_string_and_arg_to_expected_value", + [ + ( + "--arg_str test --arg_list x y z", + {"arg_str": "test", "arg_bool": True, "arg_list": ["x", "y", "z"]}, + ), + ( + "--arg_str test --arg_list x y z --arg_bool", + {"arg_str": "test", "arg_bool": False, "arg_list": ["x", "y", "z"]}, + ), + # The rest are invalid argument combos. This fact is indicated by the 2nd elt being a BaseException instance + ( + "--arg_list x y z --arg_bool", # Missing required arg_str + SystemExit(), + ), + ], +) +def test_convert_to_tap_class( + subclass_tap: Callable[[Any], Type[Tap]], + class_or_function: Any, + args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]], +): + args_string, arg_to_expected_value = args_string_and_arg_to_expected_value + TapSubclass = subclass_tap(class_or_function) + tap = TapSubclass(description="My description") + + # args_string is an invalid argument combo + if isinstance(arg_to_expected_value, BaseException): + expected_exception = arg_to_expected_value.__class__ + expected_error_message = str(arg_to_expected_value) or None + with pytest.raises(expected_exception=expected_exception, match=expected_error_message): + args = tap.parse_args(args_string.split()) + return + + # args_string is a valid argument combo + args = tap.parse_args(args_string.split()) + for arg, expected_value in arg_to_expected_value.items(): + assert getattr(args, arg) == expected_value + if callable(class_or_function): + result = class_or_function(**args.as_dict()) + assert result == _Args(**arg_to_expected_value) From 9163e5ca2bc09e9ae6981df30919e168f02575b4 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 00:09:54 -0800 Subject: [PATCH 15/40] add test todos --- tests/test_convert_to_tap_class.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_convert_to_tap_class.py b/tests/test_convert_to_tap_class.py index f096bcf..7cd371a 100644 --- a/tests/test_convert_to_tap_class.py +++ b/tests/test_convert_to_tap_class.py @@ -1,6 +1,7 @@ """ Tests `tap.convert_to_tap_class`. +TODO: test help message, test subclass_tap_weird, test with func_kwargs TODO: I might redesign this soon. It's not thorough yet. """ import dataclasses From c8ff11d8e8bb7bf14e440de9f2dfb79a4f607f85 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 00:21:01 -0800 Subject: [PATCH 16/40] dict -> Dict --- tap/tapify.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index c758990..45c3602 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -1,7 +1,7 @@ """Tapify module, which can initialize a class or run a function by parsing arguments from the command line.""" import dataclasses import inspect -from typing import Any, Callable, List, Optional, Sequence, Type, TypeVar, Union +from typing import Any, Callable, Dict, List, Optional, Sequence, Type, TypeVar, Union from docstring_parser import Docstring, parse @@ -71,7 +71,7 @@ def _is_pydantic_base_model(obj: Any) -> bool: def _tap_data_from_data_model( - data_model: Any, func_kwargs: dict[str, Any], param_to_description: dict[str, str] = None + data_model: Any, func_kwargs: Dict[str, Any], param_to_description: Dict[str, str] = None ) -> _TapData: """ Currently only works when `data_model` is a: @@ -151,7 +151,7 @@ def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optiona def _tap_data_from_class_or_function( - class_or_function: _ClassOrFunction, func_kwargs: dict[str, Any], param_to_description: dict[str, str] + class_or_function: _ClassOrFunction, func_kwargs: Dict[str, Any], param_to_description: Dict[str, str] ) -> _TapData: """ Note From 64820af3c0f1afc1bffc83053aaa0fbc3a907fc3 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 00:25:18 -0800 Subject: [PATCH 17/40] rename convert_to_tap_class -> to_tap_class --- demo_data_model.py | 4 ++-- tap/__init__.py | 4 ++-- tap/tapify.py | 4 ++-- ...onvert_to_tap_class.py => test_to_tap_class.py} | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) rename tests/{test_convert_to_tap_class.py => test_to_tap_class.py} (93%) diff --git a/demo_data_model.py b/demo_data_model.py index c0c403c..b13e430 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -8,7 +8,7 @@ -arg 2 """ from pydantic import BaseModel, Field -from tap import tapify, convert_to_tap_class +from tap import tapify, to_tap_class class Model(BaseModel): @@ -30,7 +30,7 @@ def to_number(string: str) -> float | int: return float(string) if "." in string else int(string) -class ModelTap(convert_to_tap_class(Model)): +class ModelTap(to_tap_class(Model)): # You can supply additional arguments here argument_with_really_long_name: float | int = 3 "This argument has a long name and will be aliased with a short one" diff --git a/tap/__init__.py b/tap/__init__.py index a5e4e74..ea7b844 100644 --- a/tap/__init__.py +++ b/tap/__init__.py @@ -1,13 +1,13 @@ from argparse import ArgumentError, ArgumentTypeError from tap._version import __version__ from tap.tap import Tap -from tap.tapify import tapify, convert_to_tap_class +from tap.tapify import tapify, to_tap_class __all__ = [ "ArgumentError", "ArgumentTypeError", "Tap", "tapify", - "convert_to_tap_class", + "to_tap_class", "__version__", ] diff --git a/tap/tapify.py b/tap/tapify.py index 45c3602..61637e8 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -241,7 +241,7 @@ def _configure(self): return ArgParser -def convert_to_tap_class(class_or_function: _ClassOrFunction, **func_kwargs) -> Type[Tap]: +def to_tap_class(class_or_function: _ClassOrFunction, **func_kwargs) -> Type[Tap]: """Creates a `Tap` class from `class_or_function`. This can be subclassed to add custom argument handling and instantiated to create a typed argument parser. @@ -275,7 +275,7 @@ def tapify( parsing the command line arguments and overwrite the function defaults but are overwritten by the parsed command line arguments. """ - # We don't directly call convert_to_tap_class b/c we need tap_data, not just tap_class + # We don't directly call to_tap_class b/c we need tap_data, not just tap_class docstring = _docstring(class_or_function) tap_data = _tap_data(class_or_function, docstring, func_kwargs) tap_class = _tap_class(tap_data.args_data) diff --git a/tests/test_convert_to_tap_class.py b/tests/test_to_tap_class.py similarity index 93% rename from tests/test_convert_to_tap_class.py rename to tests/test_to_tap_class.py index 7cd371a..3ca7039 100644 --- a/tests/test_convert_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,5 +1,5 @@ """ -Tests `tap.convert_to_tap_class`. +Tests `tap.to_tap_class`. TODO: test help message, test subclass_tap_weird, test with func_kwargs TODO: I might redesign this soon. It's not thorough yet. @@ -10,7 +10,7 @@ import pydantic import pytest -from tap import convert_to_tap_class, Tap +from tap import to_tap_class, Tap @dataclasses.dataclass @@ -79,12 +79,12 @@ class Model(pydantic.BaseModel): arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") -# Define some functions which take a class or function and calls `tap.convert_to_tap_class` on it to create a `tap.Tap` +# Define some functions which take a class or function and calls `tap.to_tap_class` on it to create a `tap.Tap` # subclass (class, not instance). Call this type of function a subclasser def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: - return convert_to_tap_class(class_or_function) # plain subclass / do nothing + return to_tap_class(class_or_function) # plain subclass / do nothing # TODO: use this. Will need to change how the test is parametrized b/c the output will depend on using @@ -93,7 +93,7 @@ def subclass_tap_weird(class_or_function): def to_number(string: str) -> float | int: return float(string) if "." in string else int(string) - class TapSubclass(convert_to_tap_class(class_or_function)): + class TapSubclass(to_tap_class(class_or_function)): # You can supply additional arguments here argument_with_really_long_name: float | int = 3 "This argument has a long name and will be aliased with a short one" @@ -122,7 +122,7 @@ def process_args(self) -> None: Class, DataclassBuiltin, DataclassBuiltin( - "convert_to_tap_class works on instances of data models (for free). It ignores the attribute values", + "to_tap_class works on instances of data models (for free). It ignores the attribute values", arg_bool=False, arg_list=["doesn't", "matter"], ), @@ -150,7 +150,7 @@ def process_args(self) -> None: ), ], ) -def test_convert_to_tap_class( +def test_to_tap_class( subclass_tap: Callable[[Any], Type[Tap]], class_or_function: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]], From 99ea5510c747c991e673232797adfa7c5dbd6173 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 03:52:28 -0800 Subject: [PATCH 18/40] lingering pipe --- tests/test_to_tap_class.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 3ca7039..7e68970 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -90,12 +90,12 @@ def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: # TODO: use this. Will need to change how the test is parametrized b/c the output will depend on using # subclass_tap_simple vs subclass_tap_weird def subclass_tap_weird(class_or_function): - def to_number(string: str) -> float | int: + def to_number(string: str) -> Union[float, int]: return float(string) if "." in string else int(string) class TapSubclass(to_tap_class(class_or_function)): # You can supply additional arguments here - argument_with_really_long_name: float | int = 3 + argument_with_really_long_name: Union[float, int] = 3 "This argument has a long name and will be aliased with a short one" def configure(self) -> None: From 1b28e879da6ecd04935704153054d0f88f11acf1 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 04:00:34 -0800 Subject: [PATCH 19/40] fix comment --- tests/test_to_tap_class.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 7e68970..23e0351 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -80,7 +80,7 @@ class Model(pydantic.BaseModel): # Define some functions which take a class or function and calls `tap.to_tap_class` on it to create a `tap.Tap` -# subclass (class, not instance). Call this type of function a subclasser +# subclass (class, not instance) def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: @@ -114,7 +114,7 @@ def process_args(self) -> None: return TapSubclass -@pytest.mark.parametrize("subclass_tap", [subclass_tap_simple]) +@pytest.mark.parametrize("subclasser", [subclass_tap_simple]) @pytest.mark.parametrize( "class_or_function", [ @@ -151,12 +151,12 @@ def process_args(self) -> None: ], ) def test_to_tap_class( - subclass_tap: Callable[[Any], Type[Tap]], + subclasser: Callable[[Any], Type[Tap]], class_or_function: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]], ): args_string, arg_to_expected_value = args_string_and_arg_to_expected_value - TapSubclass = subclass_tap(class_or_function) + TapSubclass = subclasser(class_or_function) tap = TapSubclass(description="My description") # args_string is an invalid argument combo From 0dcb8646fb319129143889083366a2ea6b4fd8ad Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 20:21:33 -0800 Subject: [PATCH 20/40] test more complex subclasser --- tests/test_to_tap_class.py | 193 ++++++++++++++++++++++++++----------- 1 file changed, 136 insertions(+), 57 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 23e0351..7a62ea2 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,8 +1,7 @@ """ Tests `tap.to_tap_class`. -TODO: test help message, test subclass_tap_weird, test with func_kwargs -TODO: I might redesign this soon. It's not thorough yet. +TODO: test help message, test with func_kwargs """ import dataclasses from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union @@ -17,7 +16,7 @@ class _Args: """These are the argument names which every type of class or function must contain.""" - arg_str: str = dataclasses.field(metadata=dict(description="some string")) + arg_int: int = dataclasses.field(metadata=dict(description="some integer")) arg_bool: bool = True arg_list: Optional[List[str]] = dataclasses.field(default=None, metadata=dict(description="some list of strings")) @@ -26,7 +25,7 @@ def _monkeypatch_eq(cls): """Monkey-patches `cls.__eq__` to check that the attribute values are equal to a dataclass representation of them""" def _equality(self, other: _Args) -> bool: - return _Args(self.arg_str, self.arg_bool, self.arg_list) == other + return _Args(self.arg_int, self.arg_bool, self.arg_list) == other cls.__eq__ = _equality return cls @@ -36,9 +35,9 @@ def _equality(self, other: _Args) -> bool: # if not required) -def function(arg_str: str, arg_bool: bool = True, arg_list: Optional[List[str]] = None) -> _Args: +def function(arg_int: int, arg_bool: bool = True, arg_list: Optional[List[str]] = None) -> _Args: """ - :param arg_str: some string + :param arg_int: some integer :param arg_list: some list of strings """ return _Args(*locals().values()) @@ -46,12 +45,12 @@ def function(arg_str: str, arg_bool: bool = True, arg_list: Optional[List[str]] @_monkeypatch_eq class Class: - def __init__(self, arg_str: str, arg_bool: bool = True, arg_list: Optional[List[str]] = None): + def __init__(self, arg_int: int, arg_bool: bool = True, arg_list: Optional[List[str]] = None): """ - :param arg_str: some string + :param arg_int: some integer :param arg_list: some list of strings """ - self.arg_str = arg_str + self.arg_int = arg_int self.arg_bool = arg_bool self.arg_list = arg_list @@ -65,7 +64,7 @@ def __init__(self, arg_str: str, arg_bool: bool = True, arg_list: Optional[List[ class DataclassPydantic: """Dataclass (pydantic)""" - arg_str: str = pydantic.dataclasses.Field(description="some string") + arg_int: int = pydantic.dataclasses.Field(description="some integer") arg_bool: bool = pydantic.dataclasses.Field(default=True) arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") @@ -74,22 +73,52 @@ class DataclassPydantic: class Model(pydantic.BaseModel): """Pydantic model""" - arg_str: str = pydantic.Field(description="some string") + arg_int: int = pydantic.Field(description="some integer") arg_bool: bool = pydantic.Field(default=True) arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") +@pytest.fixture( + scope="module", + params=[ + function, + Class, + DataclassBuiltin, + DataclassBuiltin( + "to_tap_class will work on instances of data models (for free). It ignores the attribute values", + arg_bool=False, + arg_list=["doesn't", "matter"], + ), + DataclassPydantic, + DataclassPydantic(arg_int=1_000, arg_bool=False, arg_list=[]), + Model, + Model(arg_int=1_000, arg_bool=False, arg_list=[]), + ], +) +def data_model(request: pytest.FixtureRequest): + """ + Same as class_or_function. Only difference is that data_model is parametrized while class_or_function is not. + """ + return request.param + + # Define some functions which take a class or function and calls `tap.to_tap_class` on it to create a `tap.Tap` # subclass (class, not instance) def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: - return to_tap_class(class_or_function) # plain subclass / do nothing + """ + Plain subclass, does nothing extra. + """ + return to_tap_class(class_or_function) -# TODO: use this. Will need to change how the test is parametrized b/c the output will depend on using -# subclass_tap_simple vs subclass_tap_weird -def subclass_tap_weird(class_or_function): +def subclass_tap_complex(class_or_function): + """ + It's conceivable that someone has a data model, but they want to add more arguments or handling when running a + script. + """ + def to_number(string: str) -> Union[float, int]: return float(string) if "." in string else int(string) @@ -105,62 +134,32 @@ def configure(self) -> None: def process_args(self) -> None: # You can still validate and modify arguments if self.argument_with_really_long_name > 4: - raise ValueError("nope") + raise ValueError("argument_with_really_long_name cannot be > 4") # No auto-complete (and other niceties) for the super class attributes b/c this is a dynamic subclass. Sorry - if self.arg_bool: - self.arg_str += " processed" + if self.arg_bool and self.arg_list is not None: + self.arg_list.append("processed") return TapSubclass -@pytest.mark.parametrize("subclasser", [subclass_tap_simple]) -@pytest.mark.parametrize( - "class_or_function", - [ - function, - Class, - DataclassBuiltin, - DataclassBuiltin( - "to_tap_class works on instances of data models (for free). It ignores the attribute values", - arg_bool=False, - arg_list=["doesn't", "matter"], - ), - DataclassPydantic, - DataclassPydantic(arg_str="...", arg_bool=False, arg_list=[]), - Model, - Model(arg_str="...", arg_bool=False, arg_list=[]), - ], -) -@pytest.mark.parametrize( - "args_string_and_arg_to_expected_value", - [ - ( - "--arg_str test --arg_list x y z", - {"arg_str": "test", "arg_bool": True, "arg_list": ["x", "y", "z"]}, - ), - ( - "--arg_str test --arg_list x y z --arg_bool", - {"arg_str": "test", "arg_bool": False, "arg_list": ["x", "y", "z"]}, - ), - # The rest are invalid argument combos. This fact is indicated by the 2nd elt being a BaseException instance - ( - "--arg_list x y z --arg_bool", # Missing required arg_str - SystemExit(), - ), - ], -) -def test_to_tap_class( +# Test that the subclasser parses the args correctly or raises the correct error. +# The subclassers are tested separately b/c the parametrizaiton of args_string_and_arg_to_expected_value depends on the +# subclasser. + + +def _test_subclasser( subclasser: Callable[[Any], Type[Tap]], class_or_function: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]], + test_call: bool = True, ): args_string, arg_to_expected_value = args_string_and_arg_to_expected_value TapSubclass = subclasser(class_or_function) tap = TapSubclass(description="My description") - # args_string is an invalid argument combo if isinstance(arg_to_expected_value, BaseException): + # args_string is an invalid argument combo expected_exception = arg_to_expected_value.__class__ expected_error_message = str(arg_to_expected_value) or None with pytest.raises(expected_exception=expected_exception, match=expected_error_message): @@ -171,6 +170,86 @@ def test_to_tap_class( args = tap.parse_args(args_string.split()) for arg, expected_value in arg_to_expected_value.items(): assert getattr(args, arg) == expected_value - if callable(class_or_function): + if test_call and callable(class_or_function): result = class_or_function(**args.as_dict()) assert result == _Args(**arg_to_expected_value) + + +@pytest.mark.parametrize( + "args_string_and_arg_to_expected_value", + [ + ( + "--arg_int 1 --arg_list x y z", + {"arg_int": 1, "arg_bool": True, "arg_list": ["x", "y", "z"]}, + ), + ( + "--arg_int 1 --arg_list x y z --arg_bool", + {"arg_int": 1, "arg_bool": False, "arg_list": ["x", "y", "z"]}, + ), + # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance + ( + "--arg_list x y z --arg_bool", # Missing required arg_int + SystemExit(), # TODO: figure out how to get argparse's error message and test that it matches + ), + ( + "--arg_int not_an_int --arg_list x y z --arg_bool", # Wrong type arg_int + SystemExit(), + ), + ], +) +def test_subclass_tap_simple( + data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] +): + _test_subclasser(subclass_tap_simple, data_model, args_string_and_arg_to_expected_value) + + +@pytest.mark.parametrize( + "args_string_and_arg_to_expected_value", + [ + ( + "--arg_int 1 --arg_list x y z", + { + "arg_int": 1, + "arg_bool": True, + "arg_list": ["x", "y", "z", "processed"], + "argument_with_really_long_name": 3, + }, + ), + ( + "--arg_int 1 --arg_list x y z -arg 2", + { + "arg_int": 1, + "arg_bool": True, + "arg_list": ["x", "y", "z", "processed"], + "argument_with_really_long_name": 2, + }, + ), + ( + "--arg_int 1 --arg_bool --arg_list x y z --argument_with_really_long_name 2.3", + { + "arg_int": 1, + "arg_bool": False, + "arg_list": ["x", "y", "z"], + "argument_with_really_long_name": 2.3, + }, + ), + # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance + ( + "--arg_list x y z --arg_bool", # Missing required arg_int + SystemExit(), + ), + ( + "--arg_int 1 --arg_list x y z -arg not_a_float_or_int", # Wrong type + SystemExit(), + ), + ( + "--arg_int 1 --arg_list x y z -arg 5", # Wrong value arg (aliases argument_with_really_long_name) + ValueError("argument_with_really_long_name cannot be > 4"), + ), + ], +) +def test_subclass_tap_complex( + data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] +): + # Currently setting test_call=False b/c all data models except the pydantic Model don't accept extra args + _test_subclasser(subclass_tap_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) From d3993708d6ca1ba31e503d7dc1a04b7ca708c686 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 20:24:22 -0800 Subject: [PATCH 21/40] update demo --- demo_data_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index b13e430..d9c5c94 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -2,7 +2,7 @@ Example: python demo_data_model.py \ ---arg_str test \ +--arg_int 1 \ --arg_list x y z \ --arg_bool \ -arg 2 @@ -16,7 +16,7 @@ class Model(BaseModel): My Pydantic Model which contains script args. """ - arg_str: str = Field(description="hello") + arg_int: int = Field(description="hello") arg_bool: bool = Field(default=True, description=None) arg_list: list[str] | None = Field(default=None, description="optional list") @@ -46,8 +46,8 @@ def process_args(self) -> None: raise ValueError("nope") # No auto-complete (and other niceties) for the super class attributes b/c this is a dynamic subclass. Sorry - if self.arg_bool: - self.arg_str += " processed" + if self.arg_bool and self.arg_list is not None: + self.arg_list.append("processed") if __name__ == "__main__": From 43d0d4f2747ea0230de29e94462a9c104db75e17 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 20:25:43 -0800 Subject: [PATCH 22/40] std docstrings --- tests/test_to_tap_class.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 7a62ea2..4c3d226 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -14,7 +14,9 @@ @dataclasses.dataclass class _Args: - """These are the argument names which every type of class or function must contain.""" + """ + These are the argument names which every type of class or function must contain. + """ arg_int: int = dataclasses.field(metadata=dict(description="some integer")) arg_bool: bool = True @@ -22,7 +24,9 @@ class _Args: def _monkeypatch_eq(cls): - """Monkey-patches `cls.__eq__` to check that the attribute values are equal to a dataclass representation of them""" + """ + Monkey-patches `cls.__eq__` to check that the attribute values are equal to a dataclass representation of them. + """ def _equality(self, other: _Args) -> bool: return _Args(self.arg_int, self.arg_bool, self.arg_list) == other @@ -56,13 +60,14 @@ def __init__(self, arg_int: int, arg_bool: bool = True, arg_list: Optional[List[ DataclassBuiltin = _Args -"""Dataclass (builtin)""" @_monkeypatch_eq @pydantic.dataclasses.dataclass class DataclassPydantic: - """Dataclass (pydantic)""" + """ + Dataclass (pydantic) + """ arg_int: int = pydantic.dataclasses.Field(description="some integer") arg_bool: bool = pydantic.dataclasses.Field(default=True) @@ -71,7 +76,9 @@ class DataclassPydantic: @_monkeypatch_eq class Model(pydantic.BaseModel): - """Pydantic model""" + """ + Pydantic model + """ arg_int: int = pydantic.Field(description="some integer") arg_bool: bool = pydantic.Field(default=True) From 368dcf703db60140ae8467b8e03271f675b612e7 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 21:23:11 -0800 Subject: [PATCH 23/40] test help message --- demo_data_model.py | 8 +-- tests/test_to_tap_class.py | 99 +++++++++++++++++++++++++++++++------- 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index d9c5c94..3affa31 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -16,9 +16,9 @@ class Model(BaseModel): My Pydantic Model which contains script args. """ - arg_int: int = Field(description="hello") - arg_bool: bool = Field(default=True, description=None) - arg_list: list[str] | None = Field(default=None, description="optional list") + arg_int: int = Field(description="some integer") + arg_bool: bool = Field(default=True) + arg_list: list[str] | None = Field(default=None, description="some list of strings") def main(model: Model) -> None: @@ -52,7 +52,7 @@ def process_args(self) -> None: if __name__ == "__main__": # You don't have to subclass tap_class_from_data_model(Model) if you just want a plain argument parser: - # ModelTap = tap_class_from_data_model(Model) + # ModelTap = to_tap_class(Model) args = ModelTap(description="Script description").parse_args() print("Parsed args:") print(args) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 4c3d226..f78b3e7 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,9 +1,12 @@ """ Tests `tap.to_tap_class`. -TODO: test help message, test with func_kwargs +TODO: test with func_kwargs """ +from contextlib import redirect_stdout import dataclasses +import io +import re from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union import pydantic @@ -15,7 +18,7 @@ @dataclasses.dataclass class _Args: """ - These are the argument names which every type of class or function must contain. + These are the arguments which every type of class or function must contain. """ arg_int: int = dataclasses.field(metadata=dict(description="some integer")) @@ -29,7 +32,7 @@ def _monkeypatch_eq(cls): """ def _equality(self, other: _Args) -> bool: - return _Args(self.arg_int, self.arg_bool, self.arg_list) == other + return _Args(self.arg_int, arg_bool=self.arg_bool, arg_list=self.arg_list) == other cls.__eq__ = _equality return cls @@ -44,7 +47,7 @@ def function(arg_int: int, arg_bool: bool = True, arg_list: Optional[List[str]] :param arg_int: some integer :param arg_list: some list of strings """ - return _Args(*locals().values()) + return _Args(arg_int, arg_bool=arg_bool, arg_list=arg_list) @_monkeypatch_eq @@ -92,19 +95,17 @@ class Model(pydantic.BaseModel): Class, DataclassBuiltin, DataclassBuiltin( - "to_tap_class will work on instances of data models (for free). It ignores the attribute values", - arg_bool=False, - arg_list=["doesn't", "matter"], - ), + 1, arg_bool=False, arg_list=["doesn't", "matter"] + ), # to_tap_class also works on instances of data models. It ignores the attribute values DataclassPydantic, - DataclassPydantic(arg_int=1_000, arg_bool=False, arg_list=[]), + DataclassPydantic(arg_int=1_000, arg_bool=False, arg_list=None), Model, - Model(arg_int=1_000, arg_bool=False, arg_list=[]), + Model(arg_int=1, arg_bool=True, arg_list=["not", "used"]), ], ) def data_model(request: pytest.FixtureRequest): """ - Same as class_or_function. Only difference is that data_model is parametrized while class_or_function is not. + Same meaning as class_or_function. Only difference is that data_model is parametrized. """ return request.param @@ -113,14 +114,14 @@ def data_model(request: pytest.FixtureRequest): # subclass (class, not instance) -def subclass_tap_simple(class_or_function: Any) -> Type[Tap]: +def subclasser_simple(class_or_function: Any) -> Type[Tap]: """ Plain subclass, does nothing extra. """ return to_tap_class(class_or_function) -def subclass_tap_complex(class_or_function): +def subclasser_complex(class_or_function): """ It's conceivable that someone has a data model, but they want to add more arguments or handling when running a script. @@ -163,7 +164,7 @@ def _test_subclasser( ): args_string, arg_to_expected_value = args_string_and_arg_to_expected_value TapSubclass = subclasser(class_or_function) - tap = TapSubclass(description="My description") + tap = TapSubclass(description="Script description") if isinstance(arg_to_expected_value, BaseException): # args_string is an invalid argument combo @@ -174,6 +175,7 @@ def _test_subclasser( return # args_string is a valid argument combo + # Test that parsing works correctly args = tap.parse_args(args_string.split()) for arg, expected_value in arg_to_expected_value.items(): assert getattr(args, arg) == expected_value @@ -182,6 +184,27 @@ def _test_subclasser( assert result == _Args(**arg_to_expected_value) +def _test_subclasser_help_message( + subclasser: Callable[[Any], Type[Tap]], class_or_function: Any, description: str, help_message_expected: str +): + def replace_whitespace(string: str): + # Replace all whitespaces with a single space + # FYI this line was written by an LLM: + return re.sub(r"\s+", " ", string).strip() + + TapSubclass = subclasser(class_or_function) + tap = TapSubclass(description=description) + + f = io.StringIO() + with redirect_stdout(f): + with pytest.raises(SystemExit): + tap.parse_args(["-h"]) + + help_message = f.getvalue() + # Standardize to ignore trivial differences due to terminal settings + assert replace_whitespace(help_message) == replace_whitespace(help_message_expected) + + @pytest.mark.parametrize( "args_string_and_arg_to_expected_value", [ @@ -204,10 +227,29 @@ def _test_subclasser( ), ], ) -def test_subclass_tap_simple( +def test_subclasser_simple( data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] ): - _test_subclasser(subclass_tap_simple, data_model, args_string_and_arg_to_expected_value) + _test_subclasser(subclasser_simple, data_model, args_string_and_arg_to_expected_value) + + +def test_subclasser_simple_help_message(data_model: Any): + description = "Script description" + help_message_expected = f""" +usage: pytest --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST ...]] [-h] + +{description} + +options: + --arg_int ARG_INT (int, required) some integer + --arg_bool (bool, default=True) + --arg_list [ARG_LIST ...] + (Optional[List[str]], default=None) some list of strings + -h, --help show this help message and exit +""".lstrip( + "\n" + ) + _test_subclasser_help_message(subclasser_simple, data_model, description, help_message_expected) @pytest.mark.parametrize( @@ -255,8 +297,29 @@ def test_subclass_tap_simple( ), ], ) -def test_subclass_tap_complex( +def test_subclasser_complex( data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] ): # Currently setting test_call=False b/c all data models except the pydantic Model don't accept extra args - _test_subclasser(subclass_tap_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) + _test_subclasser(subclasser_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) + + +def test_subclasser_complex_help_message(data_model: Any): + description = "Script description" + help_message_expected = f""" +usage: pytest [-arg ARGUMENT_WITH_REALLY_LONG_NAME] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST ...]] [-h] + +{description} + +options: + -arg ARGUMENT_WITH_REALLY_LONG_NAME, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME + (Union[float, int], default=3) This argument has a long name and will be aliased with a short one + --arg_int ARG_INT (int, required) some integer + --arg_bool (bool, default=True) + --arg_list [ARG_LIST ...] + (Optional[List[str]], default=None) some list of strings + -h, --help show this help message and exit +""".lstrip( + "\n" + ) + _test_subclasser_help_message(subclasser_complex, data_model, description, help_message_expected) From 0efc236d527999de2b16b987fc073abbac560294 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 21:25:41 -0800 Subject: [PATCH 24/40] test arg_list optional --- tests/test_to_tap_class.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index f78b3e7..bcae4ae 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -213,8 +213,8 @@ def replace_whitespace(string: str): {"arg_int": 1, "arg_bool": True, "arg_list": ["x", "y", "z"]}, ), ( - "--arg_int 1 --arg_list x y z --arg_bool", - {"arg_int": 1, "arg_bool": False, "arg_list": ["x", "y", "z"]}, + "--arg_int 1 --arg_bool", + {"arg_int": 1, "arg_bool": False, "arg_list": None}, ), # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance ( @@ -274,11 +274,11 @@ def test_subclasser_simple_help_message(data_model: Any): }, ), ( - "--arg_int 1 --arg_bool --arg_list x y z --argument_with_really_long_name 2.3", + "--arg_int 1 --arg_bool --argument_with_really_long_name 2.3", { "arg_int": 1, "arg_bool": False, - "arg_list": ["x", "y", "z"], + "arg_list": None, "argument_with_really_long_name": 2.3, }, ), From 5dbe6dddcc0aeb4a9d711e7c8b3828d15969ffa2 Mon Sep 17 00:00:00 2001 From: kddubey Date: Fri, 19 Jan 2024 21:44:21 -0800 Subject: [PATCH 25/40] no func_kwargs for to_tap_class --- demo_data_model.py | 2 +- tap/tapify.py | 9 +++------ tests/test_to_tap_class.py | 2 -- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index 3affa31..579527c 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -5,7 +5,7 @@ --arg_int 1 \ --arg_list x y z \ --arg_bool \ --arg 2 +-arg 3.14 """ from pydantic import BaseModel, Field from tap import tapify, to_tap_class diff --git a/tap/tapify.py b/tap/tapify.py index 61637e8..961bf1f 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -113,7 +113,6 @@ def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optiona known_only = False elif _is_pydantic_base_model(data_model): name_to_field = data_model.model_fields - # TODO: understand current behavior of extra arg handling for Pydantic models through tapify is_extra_ok = data_model.model_config.get("extra", "ignore") != "forbid" has_kwargs = is_extra_ok known_only = is_extra_ok @@ -241,17 +240,15 @@ def _configure(self): return ArgParser -def to_tap_class(class_or_function: _ClassOrFunction, **func_kwargs) -> Type[Tap]: +def to_tap_class(class_or_function: _ClassOrFunction) -> Type[Tap]: """Creates a `Tap` class from `class_or_function`. This can be subclassed to add custom argument handling and instantiated to create a typed argument parser. :param class_or_function: The class or function to run with the provided arguments. - :param func_kwargs: Additional keyword arguments for the function. These act as default values when - parsing the command line arguments and overwrite the function defaults but - are overwritten by the parsed command line arguments. """ + # TODO: add func_kwargs docstring = _docstring(class_or_function) - tap_data = _tap_data(class_or_function, docstring, func_kwargs) + tap_data = _tap_data(class_or_function, docstring, {}) return _tap_class(tap_data.args_data) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index bcae4ae..de24177 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,7 +1,5 @@ """ Tests `tap.to_tap_class`. - -TODO: test with func_kwargs """ from contextlib import redirect_stdout import dataclasses From e46120c362bb9d762a86cfc00036d42492d627e8 Mon Sep 17 00:00:00 2001 From: kddubey Date: Sat, 20 Jan 2024 00:45:09 -0800 Subject: [PATCH 26/40] fix for python<=3.10 methinks --- tap/tapify.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index 961bf1f..e01e8b9 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -10,13 +10,15 @@ except ModuleNotFoundError: BaseModel = type("BaseModel", (object,), {}) _PydanticField = type("_PydanticField", (object,), {}) + _PYDANTIC_FIELD_TYPES = () else: from pydantic import BaseModel from pydantic.fields import FieldInfo as PydanticFieldBaseModel from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass _PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] - + # typing.get_args(_PydanticField) is the empty tuple for some reason. Just repeat + _PYDANTIC_FIELD_TYPES = (PydanticFieldBaseModel, PydanticFieldDataclass) from tap import Tap @@ -132,11 +134,11 @@ def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optiona # Idiosyncrasy: if a pydantic Field is used in a pydantic dataclass, then field.default is a FieldInfo # object instead of the field's default value. Furthermore, field.annotation is always NoneType. Luckily, # the actual type of the field is stored in field.type - if isinstance(field.default, _PydanticField): + if isinstance(field.default, _PYDANTIC_FIELD_TYPES): arg_data = arg_data_from_pydantic(name, field.default, annotation=field.type) else: arg_data = arg_data_from_dataclass(name, field) - elif isinstance(field, _PydanticField): + elif isinstance(field, _PYDANTIC_FIELD_TYPES): arg_data = arg_data_from_pydantic(name, field) else: raise TypeError(f"Each field must be a dataclass or Pydantic field. Got {type(field)}") From bda73c08cc50742fa4bd49fa067352328c5d507f Mon Sep 17 00:00:00 2001 From: kddubey Date: Sat, 20 Jan 2024 04:07:25 -0800 Subject: [PATCH 27/40] pydantic v1 wackiness --- demo_data_model.py | 27 +++++++++----- tap/tapify.py | 58 +++++++++++++++++++++--------- tests/test_tapify.py | 3 ++ tests/test_to_tap_class.py | 73 +++++++++++++++++++++++++++++--------- 4 files changed, 121 insertions(+), 40 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index 579527c..ce39c1b 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -1,12 +1,23 @@ """ -Example: +Works for Pydantic v1 and v2. + +Example commands: + +python demo_data_model.py -h python demo_data_model.py \ ---arg_int 1 \ ---arg_list x y z \ ---arg_bool \ --arg 3.14 + --arg_int 1 \ + --arg_list x y z \ + --arg_bool \ + -arg 3.14 + +python demo_data_model.py \ + --arg_int 1 \ + --arg_list x y z \ + -arg 3 """ +from typing import List, Optional, Union + from pydantic import BaseModel, Field from tap import tapify, to_tap_class @@ -18,7 +29,7 @@ class Model(BaseModel): arg_int: int = Field(description="some integer") arg_bool: bool = Field(default=True) - arg_list: list[str] | None = Field(default=None, description="some list of strings") + arg_list: Optional[List[str]] = Field(default=None, description="some list of strings") def main(model: Model) -> None: @@ -26,13 +37,13 @@ def main(model: Model) -> None: print(model) -def to_number(string: str) -> float | int: +def to_number(string: str) -> Union[float, int]: return float(string) if "." in string else int(string) class ModelTap(to_tap_class(Model)): # You can supply additional arguments here - argument_with_really_long_name: float | int = 3 + argument_with_really_long_name: Union[float, int] = 3 "This argument has a long name and will be aliased with a short one" def configure(self) -> None: diff --git a/tap/tapify.py b/tap/tapify.py index e01e8b9..2d01fa5 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -1,4 +1,9 @@ -"""Tapify module, which can initialize a class or run a function by parsing arguments from the command line.""" +""" +`tapify`: initialize a class or run a function by parsing arguments from the command line. + +`to_tap_class`: convert a class or function into a `Tap` class, which can then be subclassed to add special argument +handling +""" import dataclasses import inspect from typing import Any, Callable, Dict, List, Optional, Sequence, Type, TypeVar, Union @@ -8,16 +13,19 @@ try: import pydantic except ModuleNotFoundError: + _IS_PYDANTIC_V1 = None + # These are "empty" types. isinstance and issubclass will always be False BaseModel = type("BaseModel", (object,), {}) _PydanticField = type("_PydanticField", (object,), {}) _PYDANTIC_FIELD_TYPES = () else: + _IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" from pydantic import BaseModel from pydantic.fields import FieldInfo as PydanticFieldBaseModel from pydantic.dataclasses import FieldInfo as PydanticFieldDataclass _PydanticField = Union[PydanticFieldBaseModel, PydanticFieldDataclass] - # typing.get_args(_PydanticField) is the empty tuple for some reason. Just repeat + # typing.get_args(_PydanticField) is an empty tuple for some reason. Just repeat _PYDANTIC_FIELD_TYPES = (PydanticFieldBaseModel, PydanticFieldDataclass) from tap import Tap @@ -65,13 +73,21 @@ class _TapData: "If true, ignore extra arguments and only parse known arguments" -def _is_pydantic_base_model(obj: Any) -> bool: +def _is_pydantic_base_model(obj: Union[Any, Type[Any]]) -> bool: if inspect.isclass(obj): # issublcass requires that obj is a class return issubclass(obj, BaseModel) else: return isinstance(obj, BaseModel) +def _is_pydantic_dataclass(obj: Union[Any, Type[Any]]) -> bool: + if _IS_PYDANTIC_V1: + # There's no public function in v1. This is a somewhat safe but linear check + return dataclasses.is_dataclass(obj) and any(key.startswith("__pydantic") for key in obj.__dict__) + else: + return pydantic.dataclasses.is_pydantic_dataclass(obj) + + def _tap_data_from_data_model( data_model: Any, func_kwargs: Dict[str, Any], param_to_description: Dict[str, str] = None ) -> _TapData: @@ -81,8 +97,8 @@ def _tap_data_from_data_model( - Pydantic dataclass (class or instance) - Pydantic BaseModel (class or instance). - The advantage of this function over func:`_tap_data_from_class_or_function` is that descriptions are parsed, b/c - this function look at the fields of the data model. + The advantage of this function over func:`_tap_data_from_class_or_function` is that field/argument descriptions are + extracted, b/c this function look at the fields of the data model. Note ---- @@ -199,12 +215,11 @@ def _tap_data_from_class_or_function( return _TapData(args_data, has_kwargs, known_only) -def _is_data_model(obj: Any) -> bool: +def _is_data_model(obj: Union[Any, Type[Any]]) -> bool: return dataclasses.is_dataclass(obj) or _is_pydantic_base_model(obj) def _docstring(class_or_function) -> Docstring: - """Parse class or function docstring in one line""" is_function = not inspect.isclass(class_or_function) if is_function or _is_pydantic_base_model(class_or_function): doc = class_or_function.__doc__ @@ -213,13 +228,22 @@ def _docstring(class_or_function) -> Docstring: return parse(doc) -def _tap_data(class_or_function: _ClassOrFunction, docstring: Docstring, func_kwargs) -> _TapData: - param_to_description = {param.arg_name: param.description for param in docstring.params} - if _is_data_model(class_or_function): - # TODO: allow passing func_kwargs to a Pydantic BaseModel +def _tap_data(class_or_function: _ClassOrFunction, param_to_description: Dict[str, str], func_kwargs) -> _TapData: + """ + Controls how class:`_TapData` is extracted from `class_or_function`. + """ + is_pydantic_v1_data_model = _IS_PYDANTIC_V1 and ( + _is_pydantic_base_model(class_or_function) or _is_pydantic_dataclass(class_or_function) + ) + if _is_data_model(class_or_function) and not is_pydantic_v1_data_model: + # Data models from Pydantic v1 don't lend itself well to _tap_data_from_data_model. _tap_data_from_data_model + # looks at the data model's fields. In Pydantic v1, the field.type_ attribute stores the field's + # annotation/type. But (in Pydantic v1) there's a bug where field.type_ is set to the inner-most type of a + # subscripted type. For example, annotating a field with list[str] causes field.type_ to be str, not list[str]. + # To get around this, we'll extract _TapData by looking at the signature of the data model return _tap_data_from_data_model(class_or_function, func_kwargs, param_to_description) - else: - return _tap_data_from_class_or_function(class_or_function, func_kwargs, param_to_description) + # TODO: allow passing func_kwargs to a Pydantic BaseModel + return _tap_data_from_class_or_function(class_or_function, func_kwargs, param_to_description) def _tap_class(args_data: Sequence[_ArgData]) -> Type[Tap]: @@ -248,9 +272,10 @@ def to_tap_class(class_or_function: _ClassOrFunction) -> Type[Tap]: :param class_or_function: The class or function to run with the provided arguments. """ - # TODO: add func_kwargs docstring = _docstring(class_or_function) - tap_data = _tap_data(class_or_function, docstring, {}) + param_to_description = {param.arg_name: param.description for param in docstring.params} + # TODO: add func_kwargs + tap_data = _tap_data(class_or_function, param_to_description, func_kwargs={}) return _tap_class(tap_data.args_data) @@ -276,7 +301,8 @@ def tapify( """ # We don't directly call to_tap_class b/c we need tap_data, not just tap_class docstring = _docstring(class_or_function) - tap_data = _tap_data(class_or_function, docstring, func_kwargs) + param_to_description = {param.arg_name: param.description for param in docstring.params} + tap_data = _tap_data(class_or_function, param_to_description, func_kwargs) tap_class = _tap_class(tap_data.args_data) # Create a Tap object with a description from the docstring of the class or function description = "\n".join(filter(None, (docstring.short_description, docstring.long_description))) diff --git a/tests/test_tapify.py b/tests/test_tapify.py index dbd8a03..6643dcb 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -1,3 +1,6 @@ +""" +Tests `tap.tapify`. Currently requires Pydantic v2. +""" import contextlib from dataclasses import dataclass, field import io diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index de24177..c7f392c 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,10 +1,11 @@ """ -Tests `tap.to_tap_class`. +Tests `tap.to_tap_class`. This test works for Pydantic v1 and v2. """ from contextlib import redirect_stdout import dataclasses import io import re +import sys from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union import pydantic @@ -13,6 +14,14 @@ from tap import to_tap_class, Tap +_IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" + +# To properly test the help message, we need to know how argparse formats it. It changed after 3.10 +_IS_BEFORE_PY_310 = sys.version_info < (3, 10) +_OPTIONS_TITLE = "options" if not _IS_BEFORE_PY_310 else "optional arguments" +_ARG_LIST_DOTS = "..." if not _IS_BEFORE_PY_310 else "[ARG_LIST ...]" + + @dataclasses.dataclass class _Args: """ @@ -70,20 +79,51 @@ class DataclassPydantic: Dataclass (pydantic) """ + # Mixing field types should be ok arg_int: int = pydantic.dataclasses.Field(description="some integer") arg_bool: bool = pydantic.dataclasses.Field(default=True) arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") +@_monkeypatch_eq +@pydantic.dataclasses.dataclass +class DataclassPydanticV1: # for Pydantic v1 data models, we rely on the docstring to get descriptions + """ + Dataclass (pydantic v1) + + :param arg_int: some integer + :param arg_list: some list of strings + """ + + arg_int: int + arg_bool: bool = True + arg_list: Optional[List[str]] = None + + @_monkeypatch_eq class Model(pydantic.BaseModel): """ Pydantic model """ + # Mixing field types should be ok arg_int: int = pydantic.Field(description="some integer") arg_bool: bool = pydantic.Field(default=True) - arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") + arg_list: Optional[List[str]] = pydantic.dataclasses.Field(default=None, description="some list of strings") + + +@_monkeypatch_eq +class ModelV1(pydantic.BaseModel): # for Pydantic v1 data models, we rely on the docstring to get descriptions + """ + Pydantic model (pydantic v1) + + :param arg_int: some integer + :param arg_list: some list of strings + """ + + arg_int: int + arg_bool: bool = True + arg_list: Optional[List[str]] = None @pytest.fixture( @@ -93,12 +133,11 @@ class Model(pydantic.BaseModel): Class, DataclassBuiltin, DataclassBuiltin( - 1, arg_bool=False, arg_list=["doesn't", "matter"] + 1, arg_bool=False, arg_list=["these", "values", "don't", "matter"] ), # to_tap_class also works on instances of data models. It ignores the attribute values - DataclassPydantic, - DataclassPydantic(arg_int=1_000, arg_bool=False, arg_list=None), - Model, - Model(arg_int=1, arg_bool=True, arg_list=["not", "used"]), + DataclassPydantic if not _IS_PYDANTIC_V1 else DataclassPydanticV1, + Model if not _IS_PYDANTIC_V1 else ModelV1, + # We can test instances of DataclassPydantic and Model for pydantic v2 but not v1 ], ) def data_model(request: pytest.FixtureRequest): @@ -176,7 +215,7 @@ def _test_subclasser( # Test that parsing works correctly args = tap.parse_args(args_string.split()) for arg, expected_value in arg_to_expected_value.items(): - assert getattr(args, arg) == expected_value + assert getattr(args, arg) == expected_value, arg if test_call and callable(class_or_function): result = class_or_function(**args.as_dict()) assert result == _Args(**arg_to_expected_value) @@ -231,18 +270,19 @@ def test_subclasser_simple( _test_subclasser(subclasser_simple, data_model, args_string_and_arg_to_expected_value) +# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") def test_subclasser_simple_help_message(data_model: Any): description = "Script description" help_message_expected = f""" -usage: pytest --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST ...]] [-h] +usage: pytest --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] {description} -options: +{_OPTIONS_TITLE}: --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) - --arg_list [ARG_LIST ...] - (Optional[List[str]], default=None) some list of strings + --arg_list [ARG_LIST {_ARG_LIST_DOTS}] + ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings -h, --help show this help message and exit """.lstrip( "\n" @@ -302,20 +342,21 @@ def test_subclasser_complex( _test_subclasser(subclasser_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) +# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") def test_subclasser_complex_help_message(data_model: Any): description = "Script description" help_message_expected = f""" -usage: pytest [-arg ARGUMENT_WITH_REALLY_LONG_NAME] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST ...]] [-h] +usage: pytest [-arg ARGUMENT_WITH_REALLY_LONG_NAME] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] {description} -options: +{_OPTIONS_TITLE}: -arg ARGUMENT_WITH_REALLY_LONG_NAME, --argument_with_really_long_name ARGUMENT_WITH_REALLY_LONG_NAME (Union[float, int], default=3) This argument has a long name and will be aliased with a short one --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) - --arg_list [ARG_LIST ...] - (Optional[List[str]], default=None) some list of strings + --arg_list [ARG_LIST {_ARG_LIST_DOTS}] + ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings -h, --help show this help message and exit """.lstrip( "\n" From 978b97d8ce5b74e61085046b22bc5fcef4abf2f7 Mon Sep 17 00:00:00 2001 From: kddubey Date: Sat, 20 Jan 2024 14:59:26 -0800 Subject: [PATCH 28/40] fix for py39 argparse --- tests/test_to_tap_class.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index c7f392c..d349c96 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -19,7 +19,7 @@ # To properly test the help message, we need to know how argparse formats it. It changed after 3.10 _IS_BEFORE_PY_310 = sys.version_info < (3, 10) _OPTIONS_TITLE = "options" if not _IS_BEFORE_PY_310 else "optional arguments" -_ARG_LIST_DOTS = "..." if not _IS_BEFORE_PY_310 else "[ARG_LIST ...]" +_ARG_LIST_DOTS = "..." if not sys.version_info < (3, 9) else "[ARG_LIST ...]" @dataclasses.dataclass From 97f1b6cfde4daeeb24738fa3ccc3a620aed2a327 Mon Sep 17 00:00:00 2001 From: kddubey Date: Sat, 20 Jan 2024 18:28:58 -0800 Subject: [PATCH 29/40] add to readme --- README.md | 131 ++++++++++++++++++++++++++++++++++++++++++++- demo_data_model.py | 8 +-- 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 997f9a3..6d94900 100644 --- a/README.md +++ b/README.md @@ -666,7 +666,7 @@ from tap import tapify class Squarer: """Squarer with a number to square. - :param num: The number to square. + :param num: The number to square. """ num: float @@ -681,6 +681,94 @@ if __name__ == '__main__': Running `python square_dataclass.py --num -1` prints `The square of your number is 1.0.`. +
+Argument descriptions + +For dataclasses, the argument's description (which is displayed in the `-h` help message) can either be specified in the +class docstring or the field's description in `metadata`. If both are specified, the description from the docstring is +used. In the example below, the description is provided in `metadata`. + +```python +# square_dataclass.py +from dataclasses import dataclass, field + +from tap import tapify + +@dataclass +class Squarer: + """Squarer with a number to square. + """ + num: float = field(metadata={"description": "The number to square."}) + + def get_square(self) -> float: + """Get the square of the number.""" + return self.num ** 2 + +if __name__ == '__main__': + squarer = tapify(Squarer) + print(f'The square of your number is {squarer.get_square()}.') +``` + +
+ +#### Pydantic + +Pydantic [Models](https://docs.pydantic.dev/latest/concepts/models/) and +[dataclasses](https://docs.pydantic.dev/latest/concepts/dataclasses/) can be `tapify`d. + +```python +# square_dataclass.py +from pydantic import BaseModel, Field + +from tap import tapify + +class Squarer(BaseModel): + """Squarer with a number to square. + """ + num: float = Field(description="The number to square.") + + def get_square(self) -> float: + """Get the square of the number.""" + return self.num ** 2 + +if __name__ == '__main__': + squarer = tapify(Squarer) + print(f'The square of your number is {squarer.get_square()}.') +``` + +
+Argument descriptions + +For Pydantic v2 models and dataclasses, the argument's description (which is displayed in the `-h` help message) can +either be specified in the class docstring or the field's `description`. If both are specified, the description from the +docstring is used. In the example below, the description is provided in the docstring. + +For Pydantic v1 models and dataclasses, the argument's description must be provided in the class docstring: + +```python +# square_dataclass.py +from pydantic import BaseModel + +from tap import tapify + +class Squarer(BaseModel): + """Squarer with a number to square. + + :param num: The number to square. + """ + num: float + + def get_square(self) -> float: + """Get the square of the number.""" + return self.num ** 2 + +if __name__ == '__main__': + squarer = tapify(Squarer) + print(f'The square of your number is {squarer.get_square()}.') +``` + +
+ ### tapify help The help string on the command line is set based on the docstring for the function or class. For example, running `python square_function.py -h` will print: @@ -752,3 +840,44 @@ Running `python person.py --name Jesse --age 1` prints `My name is Jesse.` follo ### Explicit boolean arguments Tapify supports explicit specification of boolean arguments (see [bool](#bool) for more details). By default, `explicit_bool=False` and it can be set with `tapify(..., explicit_bool=True)`. + +## to_tap_class + +`to_tap_class` turns a function or class into a `Tap` class. The returned class can be [subclassed](#subclassing) to add +special argument behavior. For example, you can override [`configure`](#configuring-arguments) and +[`process_args`](#argument-processing). If the object can be `tapify`d, then it can be `to_tap_class`d, and vice-versa. +`to_tap_class` provides full control over argument parsing. + +### Examples + +#### Simple + +```python +# main.py +""" +My script description +""" + +from pydantic import BaseModel + +from tap import to_tap_class + +class Project(BaseModel): + package: str + is_cool: bool = True + stars: int = 5 + +if __name__ == "__main__": + ProjectTap = to_tap_class(Project) + tap = ProjectTap(description=__doc__) # from the top of this script + args = tap.parse_args() + project = Project(**args.as_dict()) + print(f"Project instance: {project}") +``` + +Running `python main.py --package tap` will print `Project instance: package='tap' is_cool=True stars=5`. + +### Complex + +Please see `demo_data_model.py` for an example of overriding [`configure`](#configuring-arguments) and +[`process_args`](#argument-processing). diff --git a/demo_data_model.py b/demo_data_model.py index ce39c1b..a87fc19 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -8,13 +8,13 @@ python demo_data_model.py \ --arg_int 1 \ --arg_list x y z \ - --arg_bool \ - -arg 3.14 + --argument_with_really_long_name 3 python demo_data_model.py \ --arg_int 1 \ --arg_list x y z \ - -arg 3 + --arg_bool \ + -arg 3.14 """ from typing import List, Optional, Union @@ -73,7 +73,7 @@ def process_args(self) -> None: main(model) -# This works but doesn't show the field description, and immediately returns a Model instance instead of a Tap class +# tapify works with Model. It immediately returns a Model instance instead of a Tap class # if __name__ == "__main__": # model = tapify(Model) # print(model) From c2c7087317bae3a30c45bca83d952081ae4be5fe Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 05:31:19 -0800 Subject: [PATCH 30/40] test subparsing --- demo_data_model.py | 22 ++++- tests/test_to_tap_class.py | 186 ++++++++++++++++++++++++++++++++++--- 2 files changed, 193 insertions(+), 15 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index a87fc19..93e6f94 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -16,10 +16,10 @@ --arg_bool \ -arg 3.14 """ -from typing import List, Optional, Union +from typing import List, Literal, Optional, Union from pydantic import BaseModel, Field -from tap import tapify, to_tap_class +from tap import tapify, to_tap_class, Tap class Model(BaseModel): @@ -61,10 +61,28 @@ def process_args(self) -> None: self.arg_list.append("processed") +# class SubparserA(Tap): +# bar: int # bar help + + +# class SubparserB(Tap): +# baz: Literal["X", "Y", "Z"] # baz help + + +# class ModelTapWithSubparsing(to_tap_class(Model)): +# foo: bool = False # foo help + +# def configure(self): +# self.add_subparsers(help="sub-command help") +# self.add_subparser("a", SubparserA, help="a help", description="Description (a)") +# self.add_subparser("b", SubparserB, help="b help") + + if __name__ == "__main__": # You don't have to subclass tap_class_from_data_model(Model) if you just want a plain argument parser: # ModelTap = to_tap_class(Model) args = ModelTap(description="Script description").parse_args() + # args = ModelTapWithSubparsing(description="Script description").parse_args() print("Parsed args:") print(args) # Run the main function. Pydantic BaseModels ignore arguments which aren't one of their fields instead of raising an diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index d349c96..7f8965b 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -6,7 +6,7 @@ import io import re import sys -from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union import pydantic import pytest @@ -16,7 +16,7 @@ _IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" -# To properly test the help message, we need to know how argparse formats it. It changed after 3.10 +# To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 _IS_BEFORE_PY_310 = sys.version_info < (3, 10) _OPTIONS_TITLE = "options" if not _IS_BEFORE_PY_310 else "optional arguments" _ARG_LIST_DOTS = "..." if not sys.version_info < (3, 9) else "[ARG_LIST ...]" @@ -188,6 +188,24 @@ def process_args(self) -> None: return TapSubclass +def subclasser_subparser(class_or_function): + class SubparserA(Tap): + bar: int # bar help + + class SubparserB(Tap): + baz: Literal["X", "Y", "Z"] # baz help + + class TapSubclass(to_tap_class(class_or_function)): + foo: bool = False # foo help + + def configure(self): + self.add_subparsers(help="sub-command help") + self.add_subparser("a", SubparserA, help="a help", description="Description (a)") + self.add_subparser("b", SubparserB, help="b help") + + return TapSubclass + + # Test that the subclasser parses the args correctly or raises the correct error. # The subclassers are tested separately b/c the parametrizaiton of args_string_and_arg_to_expected_value depends on the # subclasser. @@ -199,9 +217,16 @@ def _test_subclasser( args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]], test_call: bool = True, ): + """ + Tests that the `subclasser` converts `class_or_function` to a `Tap` class which parses the argument string + correctly. + + Setting `test_call=True` additionally tests that calling the `class_or_function` on the parsed arguments works. + """ args_string, arg_to_expected_value = args_string_and_arg_to_expected_value TapSubclass = subclasser(class_or_function) - tap = TapSubclass(description="Script description") + assert issubclass(TapSubclass, Tap) + tap = TapSubclass(description="Script description") # description is a kwarg for argparse.ArgumentParser if isinstance(arg_to_expected_value, BaseException): # args_string is an invalid argument combo @@ -214,16 +239,27 @@ def _test_subclasser( # args_string is a valid argument combo # Test that parsing works correctly args = tap.parse_args(args_string.split()) - for arg, expected_value in arg_to_expected_value.items(): - assert getattr(args, arg) == expected_value, arg + assert arg_to_expected_value == args.as_dict() if test_call and callable(class_or_function): result = class_or_function(**args.as_dict()) assert result == _Args(**arg_to_expected_value) -def _test_subclasser_help_message( - subclasser: Callable[[Any], Type[Tap]], class_or_function: Any, description: str, help_message_expected: str +def _test_subclasser_message( + subclasser: Callable[[Any], Type[Tap]], + class_or_function: Any, + message_expected: str, + description: str = "Script description", + args_string: str = "-h", ): + """ + Tests that:: + + subclasser(class_or_function)(description=description).parse_args(args_string.split()) + + outputs `message_expected` to stdout, ignoring differences in whitespaces/newlines/tabs. + """ + def replace_whitespace(string: str): # Replace all whitespaces with a single space # FYI this line was written by an LLM: @@ -235,11 +271,14 @@ def replace_whitespace(string: str): f = io.StringIO() with redirect_stdout(f): with pytest.raises(SystemExit): - tap.parse_args(["-h"]) + tap.parse_args(args_string.split()) - help_message = f.getvalue() + message = f.getvalue() # Standardize to ignore trivial differences due to terminal settings - assert replace_whitespace(help_message) == replace_whitespace(help_message_expected) + assert replace_whitespace(message) == replace_whitespace(message_expected) + + +# Test sublcasser_simple @pytest.mark.parametrize( @@ -256,7 +295,7 @@ def replace_whitespace(string: str): # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance ( "--arg_list x y z --arg_bool", # Missing required arg_int - SystemExit(), # TODO: figure out how to get argparse's error message and test that it matches + SystemExit(), # TODO: get argparse's error message and test that it matches ), ( "--arg_int not_an_int --arg_list x y z --arg_bool", # Wrong type arg_int @@ -287,7 +326,10 @@ def test_subclasser_simple_help_message(data_model: Any): """.lstrip( "\n" ) - _test_subclasser_help_message(subclasser_simple, data_model, description, help_message_expected) + _test_subclasser_message(subclasser_simple, data_model, help_message_expected, description=description) + + +# Test subclasser_complex @pytest.mark.parametrize( @@ -361,4 +403,122 @@ def test_subclasser_complex_help_message(data_model: Any): """.lstrip( "\n" ) - _test_subclasser_help_message(subclasser_complex, data_model, description, help_message_expected) + _test_subclasser_message(subclasser_complex, data_model, help_message_expected, description=description) + + +# Test subclasser_subparser + + +@pytest.mark.parametrize( + "args_string_and_arg_to_expected_value", + [ + ( + "--arg_int 1", + {"arg_int": 1, "arg_bool": True, "arg_list": None, "foo": False}, + ), + ( + "--arg_int 1 a --bar 2", + {"arg_int": 1, "arg_bool": True, "arg_list": None, "bar": 2, "foo": False}, + ), + ( + "--arg_int 1 --foo a --bar 2", + {"arg_int": 1, "arg_bool": True, "arg_list": None, "bar": 2, "foo": True}, + ), + ( + "--arg_int 1 b --baz X", + {"arg_int": 1, "arg_bool": True, "arg_list": None, "baz": "X", "foo": False}, + ), + ( + "--foo --arg_bool --arg_list x y z --arg_int 1 b --baz Y", + {"arg_int": 1, "arg_bool": False, "arg_list": ["x", "y", "z"], "baz": "Y", "foo": True}, + ), + # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance + ( + "a --bar 1", + SystemExit(), # error: the following arguments are required: --arg_int + ), + ( + "--arg_int not_an_int a --bar 1", + SystemExit(), # error: argument --arg_int: invalid int value: 'not_an_int' + ), + ( + "--arg_int 1 --baz X --foo b", + SystemExit(), # error: argument {a,b}: invalid choice: 'X' (choose from 'a', 'b') + ), + ( + "--arg_int 1 b --baz X --foo", + SystemExit(), # error: unrecognized arguments: --foo + ), + ( + "--arg_int 1 --foo b --baz A", + SystemExit(), # error: argument --baz: Value for variable "baz" must be one of ['X', 'Y', 'Z']. + ), + ], +) +def test_subclasser_subparser( + data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] +): + # Currently setting test_call=False b/c all data models except the pydantic Model don't accept extra args + _test_subclasser(subclasser_subparser, data_model, args_string_and_arg_to_expected_value, test_call=False) + + +# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") +@pytest.mark.parametrize( + "args_string_and_description_and_expected_message", + [ + ( + "-h", + "Script description", + # foo help likely missing b/c class nesting. In a demo in a Python 3.8 env, foo help appears in -h + f""" +usage: pytest [--foo] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] {{a,b}} ... + +Script description + +positional arguments: + {{a,b}} sub-command help + a a help + b b help + +{_OPTIONS_TITLE}: + --foo (bool, default=False) {'' if sys.version_info < (3, 9) else 'foo help'} + --arg_int ARG_INT (int, required) some integer + --arg_bool (bool, default=True) + --arg_list [ARG_LIST {_ARG_LIST_DOTS}] + ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings + -h, --help show this help message and exit +""", + ), + ( + "a -h", + "Description (a)", + f""" +usage: pytest a --bar BAR [-h] + +Description (a) + +{_OPTIONS_TITLE}: + --bar BAR (int, required) bar help + -h, --help show this help message and exit +""", + ), + ( + "b -h", + "", + f""" +usage: pytest b --baz {{X,Y,Z}} [-h] + +{_OPTIONS_TITLE}: + --baz {{X,Y,Z}} (Literal['X', 'Y', 'Z'], required) baz help + -h, --help show this help message and exit +""", + ), + ], +) +def test_subclasser_subparser_help_message( + data_model: Any, args_string_and_description_and_expected_message: Tuple[str, str] +): + args_string, description, expected_message = args_string_and_description_and_expected_message + _test_subclasser_message( + subclasser_subparser, data_model, expected_message, description=description, args_string=args_string + ) From 76a68f75c08115e1793acfb625e27f7d934ae013 Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 06:29:32 -0800 Subject: [PATCH 31/40] test SystemExit error message --- demo_data_model.py | 2 +- tests/test_to_tap_class.py | 71 ++++++++++++++++++++++++-------------- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/demo_data_model.py b/demo_data_model.py index 93e6f94..95925c7 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -54,7 +54,7 @@ def process_args(self) -> None: # You can still validate and modify arguments # (You should do this in the Pydantic Model. I'm just demonstrating that this functionality is still possible) if self.argument_with_really_long_name > 4: - raise ValueError("nope") + raise ValueError("argument_with_really_long_name cannot be > 4") # No auto-complete (and other niceties) for the super class attributes b/c this is a dynamic subclass. Sorry if self.arg_bool and self.arg_list is not None: diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 7f8965b..09cc553 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,7 +1,7 @@ """ Tests `tap.to_tap_class`. This test works for Pydantic v1 and v2. """ -from contextlib import redirect_stdout +from contextlib import redirect_stdout, redirect_stderr import dataclasses import io import re @@ -209,6 +209,22 @@ def configure(self): # Test that the subclasser parses the args correctly or raises the correct error. # The subclassers are tested separately b/c the parametrizaiton of args_string_and_arg_to_expected_value depends on the # subclasser. +# First, some helper functions. + + +def _test_raises_system_exit(tap: Tap, args_string: str) -> str: + is_help = ( + args_string.endswith("-h") + or args_string.endswith("--help") + or " -h " in args_string + or " --help " in args_string + ) + f = io.StringIO() + with redirect_stdout(f) if is_help else redirect_stderr(f): + with pytest.raises(SystemExit): + tap.parse_args(args_string.split()) + + return f.getvalue() def _test_subclasser( @@ -228,8 +244,13 @@ def _test_subclasser( assert issubclass(TapSubclass, Tap) tap = TapSubclass(description="Script description") # description is a kwarg for argparse.ArgumentParser + # args_string is an invalid argument combo + if isinstance(arg_to_expected_value, SystemExit): + # We need to get the error message by reading stdout + stdout = _test_raises_system_exit(tap, args_string) + assert str(arg_to_expected_value) in stdout + return if isinstance(arg_to_expected_value, BaseException): - # args_string is an invalid argument combo expected_exception = arg_to_expected_value.__class__ expected_error_message = str(arg_to_expected_value) or None with pytest.raises(expected_exception=expected_exception, match=expected_error_message): @@ -260,20 +281,12 @@ def _test_subclasser_message( outputs `message_expected` to stdout, ignoring differences in whitespaces/newlines/tabs. """ - def replace_whitespace(string: str): - # Replace all whitespaces with a single space - # FYI this line was written by an LLM: - return re.sub(r"\s+", " ", string).strip() + def replace_whitespace(string: str) -> str: + return re.sub(r"\s+", " ", string).strip() # FYI this line was written by an LLM TapSubclass = subclasser(class_or_function) tap = TapSubclass(description=description) - - f = io.StringIO() - with redirect_stdout(f): - with pytest.raises(SystemExit): - tap.parse_args(args_string.split()) - - message = f.getvalue() + message = _test_raises_system_exit(tap, args_string) # Standardize to ignore trivial differences due to terminal settings assert replace_whitespace(message) == replace_whitespace(message_expected) @@ -294,12 +307,12 @@ def replace_whitespace(string: str): ), # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance ( - "--arg_list x y z --arg_bool", # Missing required arg_int - SystemExit(), # TODO: get argparse's error message and test that it matches + "--arg_list x y z --arg_bool", + SystemExit("error: the following arguments are required: --arg_int"), ), ( - "--arg_int not_an_int --arg_list x y z --arg_bool", # Wrong type arg_int - SystemExit(), + "--arg_int not_an_int --arg_list x y z --arg_bool", + SystemExit("error: argument --arg_int: invalid int value: 'not_an_int'"), ), ], ) @@ -364,12 +377,14 @@ def test_subclasser_simple_help_message(data_model: Any): ), # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance ( - "--arg_list x y z --arg_bool", # Missing required arg_int - SystemExit(), + "--arg_list x y z --arg_bool", + SystemExit("error: the following arguments are required: --arg_int"), ), ( - "--arg_int 1 --arg_list x y z -arg not_a_float_or_int", # Wrong type - SystemExit(), + "--arg_int 1 --arg_list x y z -arg not_a_float_or_int", + SystemExit( + "error: argument -arg/--argument_with_really_long_name: invalid to_number value: 'not_a_float_or_int'" + ), ), ( "--arg_int 1 --arg_list x y z -arg 5", # Wrong value arg (aliases argument_with_really_long_name) @@ -435,23 +450,27 @@ def test_subclasser_complex_help_message(data_model: Any): # The rest are invalid argument combos, as indicated by the 2nd elt being a BaseException instance ( "a --bar 1", - SystemExit(), # error: the following arguments are required: --arg_int + SystemExit("error: the following arguments are required: --arg_int"), ), ( "--arg_int not_an_int a --bar 1", - SystemExit(), # error: argument --arg_int: invalid int value: 'not_an_int' + SystemExit("error: argument --arg_int: invalid int value: 'not_an_int'"), ), ( "--arg_int 1 --baz X --foo b", - SystemExit(), # error: argument {a,b}: invalid choice: 'X' (choose from 'a', 'b') + SystemExit( + "error: argument {a,b}: invalid choice: 'X' (choose from 'a', 'b')" + if sys.version_info >= (3, 9) + else "error: invalid choice: 'X' (choose from 'a', 'b')" + ), ), ( "--arg_int 1 b --baz X --foo", - SystemExit(), # error: unrecognized arguments: --foo + SystemExit("error: unrecognized arguments: --foo"), ), ( "--arg_int 1 --foo b --baz A", - SystemExit(), # error: argument --baz: Value for variable "baz" must be one of ['X', 'Y', 'Z']. + SystemExit("""error: argument --baz: Value for variable "baz" must be one of ['X', 'Y', 'Z']."""), ), ], ) From 15eaf3bf5bde3917c2b8b5d43fc8728e51275440 Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 06:41:32 -0800 Subject: [PATCH 32/40] stdout -> stderr for non-help --- tests/test_to_tap_class.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 09cc553..2f846e9 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -247,8 +247,8 @@ def _test_subclasser( # args_string is an invalid argument combo if isinstance(arg_to_expected_value, SystemExit): # We need to get the error message by reading stdout - stdout = _test_raises_system_exit(tap, args_string) - assert str(arg_to_expected_value) in stdout + stderr = _test_raises_system_exit(tap, args_string) + assert str(arg_to_expected_value) in stderr return if isinstance(arg_to_expected_value, BaseException): expected_exception = arg_to_expected_value.__class__ From f38fba7c4a65418496e1c8f6a7cf4ec9b148635f Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 08:18:51 -0800 Subject: [PATCH 33/40] dont require pydantic for tests to run --- tests/test_tapify.py | 698 +++++++++++++++++++++---------------- tests/test_to_tap_class.py | 106 +++--- 2 files changed, 456 insertions(+), 348 deletions(-) diff --git a/tests/test_tapify.py b/tests/test_tapify.py index 6643dcb..7d6ac0d 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -9,11 +9,17 @@ import unittest from unittest import TestCase -import pydantic - from tap import tapify +try: + import pydantic +except ModuleNotFoundError: + _IS_PYDANTIC_V1 = None +else: + _IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" + + # Suppress prints from SystemExit class DevNull: def write(self, msg): @@ -27,7 +33,7 @@ class Person: def __init__(self, name: str): self.name = name - def __str__(self) -> str: + def __repr__(self) -> str: return f"Person({self.name})" @@ -36,7 +42,7 @@ def __init__(self, problem_1: str, problem_2): self.problem_1 = problem_1 self.problem_2 = problem_2 - def __str__(self) -> str: + def __repr__(self) -> str: return f"Problems({self.problem_1}, {self.problem_2})" @@ -54,16 +60,22 @@ class PieDataclass: def __eq__(self, other: float) -> bool: return other == pie() - @pydantic.dataclasses.dataclass - class PieDataclassPydantic: - def __eq__(self, other: float) -> bool: - return other == pie() + if _IS_PYDANTIC_V1 is not None: - class PieModel(pydantic.BaseModel): - def __eq__(self, other: float) -> bool: - return other == pie() + @pydantic.dataclasses.dataclass + class PieDataclassPydantic: + def __eq__(self, other: float) -> bool: + return other == pie() + + class PieModel(pydantic.BaseModel): + def __eq__(self, other: float) -> bool: + return other == pie() - for class_or_function in [pie, Pie, PieDataclass, PieDataclassPydantic, PieModel]: + pydantic_data_models = [PieDataclassPydantic, PieModel] + else: + pydantic_data_models = [] + + for class_or_function in [pie, Pie, PieDataclass] + pydantic_data_models: self.assertEqual(tapify(class_or_function, command_line_args=[]), 3.14) def test_tapify_simple_types(self): @@ -88,28 +100,34 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - a: int - simple: str - test: float - of: float - types: bool + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.simple, self.test, self.of, self.types) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + a: int + simple: str + test: float + of: float + types: bool - class ConcatModel(pydantic.BaseModel): - a: int - simple: str - test: float - of: float - types: bool + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types) - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.simple, self.test, self.of, self.types) + class ConcatModel(pydantic.BaseModel): + a: int + simple: str + test: float + of: float + types: bool + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--of", "2.718", "--types"], @@ -142,30 +160,37 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - a: int - simple: str - test: float - of: float = -0.3 - types: bool = pydantic.dataclasses.Field(False) - wow: str = pydantic.Field("abc") # mixing field types should be ok - - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) - - class ConcatModel(pydantic.BaseModel): - a: int - simple: str - test: float - of: float = -0.3 - types: bool = pydantic.Field(False) - wow: str = pydantic.dataclasses.Field("abc") # mixing field types should be ok - - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) - - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + if _IS_PYDANTIC_V1 is not None: + + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + a: int + simple: str + test: float + of: float = -0.3 + types: bool = False + wow: str = "abc" + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) + + class ConcatModel(pydantic.BaseModel): + a: int + simple: str + test: float + of: float = -0.3 + types: bool = False + wow: str = "abc" + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.simple, self.test, self.of, self.types, self.wow) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: + print(class_or_function.__name__) output = tapify( class_or_function, command_line_args=["--a", "1", "--simple", "simple", "--test", "3.14", "--types", "--wow", "wee"], @@ -193,26 +218,38 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person - class ConcatDataclassPydantic: - complexity: List[str] - requires: Tuple[int, int] - intelligence: Person + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: List[str] + requires: Tuple[int, int] + intelligence: Person - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) - complexity: List[str] - requires: Tuple[int, int] - intelligence: Person + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence) + class Config: + arbitrary_types_allowed = True # for Person + + else: + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + complexity: List[str] + requires: Tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=[ @@ -253,26 +290,38 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence) - @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person - class ConcatDataclassPydantic: - complexity: list[int] - requires: tuple[int, int] - intelligence: Person + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: list[int] + requires: tuple[int, int] + intelligence: Person - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) - complexity: list[int] - requires: tuple[int, int] - intelligence: Person + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence) + class Config: + arbitrary_types_allowed = True # for Person + + else: + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + + complexity: list[int] + requires: tuple[int, int] + intelligence: Person + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=[ @@ -331,30 +380,42 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) - @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person - class ConcatDataclassPydantic: - complexity: List[str] - requires: Tuple[int, int] = (2, 5) - intelligence: Person = Person("kyle") - maybe: Optional[str] = None - possibly: Optional[str] = None + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) # for Person + class ConcatDataclassPydantic: + complexity: List[str] + requires: Tuple[int, int] = (2, 5) + intelligence: Person = Person("kyle") + maybe: Optional[str] = None + possibly: Optional[str] = None - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) - complexity: List[str] - requires: Tuple[int, int] = (2, 5) - intelligence: Person = Person("kyle") - maybe: Optional[str] = None - possibly: Optional[str] = None + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + class Config: + arbitrary_types_allowed = True # for Person + + else: + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Person - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + complexity: List[str] + requires: Tuple[int, int] = (2, 5) + intelligence: Person = Person("kyle") + maybe: Optional[str] = None + possibly: Optional[str] = None + + def __eq__(self, other: str) -> bool: + return other == concat(self.complexity, self.requires, self.intelligence, self.maybe, self.possibly) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=[ @@ -392,24 +453,30 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.many, self.args) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - so: int - many: float - args: str + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.many, self.args) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + many: float + args: str - class ConcatModel(pydantic.BaseModel): - so: int - many: float - args: str + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.many, self.args) - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.many, self.args) + class ConcatModel(pydantic.BaseModel): + so: int + many: float + args: str + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.many, self.args) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--many", "9.3"]) @@ -432,24 +499,36 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - so: int - few: float + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.few) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + few: float - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) - so: int - few: float + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.few) + class Config: + extra = pydantic.Extra.forbid # by default, pydantic ignores extra arguments + + else: + model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"]) @@ -472,22 +551,28 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - so: int - few: float + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.few) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + so: int + few: float - class ConcatModel(pydantic.BaseModel): - so: int - few: float + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) - def __eq__(self, other: str) -> bool: - return other == concat(self.so, self.few) + class ConcatModel(pydantic.BaseModel): + so: int + few: float + + def __eq__(self, other: str) -> bool: + return other == concat(self.so, self.few) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=["--so", "23", "--few", "9.3", "--args", "wow"], known_only=True ) @@ -517,30 +602,36 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - i: int - like: float - k: int - w: str = "w" - args: str = "argy" - always: bool = False + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False - class ConcatModel(pydantic.BaseModel): - i: int - like: float - k: int - w: str = "w" - args: str = "argy" - always: bool = False + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - def __eq__(self, other: str) -> bool: - return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + class ConcatModel(pydantic.BaseModel): + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=[ @@ -583,32 +674,44 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - i: int - like: float - k: int - w: str = "w" - args: str = "argy" - always: bool = False + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) - i: int - like: float - k: int - w: str = "w" - args: str = "argy" - always: bool = False + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + class Config: + extra = pydantic.Extra.forbid # by default, pydantic ignores extra arguments + + else: + model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + i: int + like: float + k: int + w: str = "w" + args: str = "argy" + always: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: with self.assertRaises(ValueError): tapify( class_or_function, @@ -644,22 +747,34 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.problems) - @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) - class ConcatDataclassPydantic: - problems: Problems + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.problems) + @pydantic.dataclasses.dataclass(config=dict(arbitrary_types_allowed=True)) + class ConcatDataclassPydantic: + problems: Problems - class ConcatModel(pydantic.BaseModel): - model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Problems + def __eq__(self, other: str) -> bool: + return other == concat(self.problems) - problems: Problems + class ConcatModel(pydantic.BaseModel): + if _IS_PYDANTIC_V1: - def __eq__(self, other: str) -> bool: - return other == concat(self.problems) + class Config: + arbitrary_types_allowed = True # for Problems + + else: + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) # for Problems + + problems: Problems + + def __eq__(self, other: str) -> bool: + return other == concat(self.problems) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify(class_or_function, command_line_args=[], problems=Problems("oh", "no!")) self.assertEqual(output, "Problems(oh, no!)") @@ -703,34 +818,40 @@ def __eq__(self, other: str) -> bool: self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 ) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - untyped_1: Any - typed_1: int - untyped_2: Any = 5 - typed_2: str = "now" - untyped_3: Any = "hi" - typed_3: bool = False - - def __eq__(self, other: str) -> bool: - return other == concat( - self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 - ) - - class ConcatModel(pydantic.BaseModel): - untyped_1: Any - typed_1: int - untyped_2: Any = 5 - typed_2: str = "now" - untyped_3: Any = "hi" - typed_3: bool = False - - def __eq__(self, other: str) -> bool: - return other == concat( - self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 - ) - - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + if _IS_PYDANTIC_V1 is not None: + + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + untyped_1: Any + typed_1: int + untyped_2: Any = 5 + typed_2: str = "now" + untyped_3: Any = "hi" + typed_3: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat( + self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 + ) + + class ConcatModel(pydantic.BaseModel): + untyped_1: Any + typed_1: int + untyped_2: Any = 5 + typed_2: str = "now" + untyped_3: Any = "hi" + typed_3: bool = False + + def __eq__(self, other: str) -> bool: + return other == concat( + self.untyped_1, self.typed_1, self.untyped_2, self.typed_2, self.untyped_3, self.typed_3 + ) + + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output = tapify( class_or_function, command_line_args=[ @@ -773,28 +894,33 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydantic: - """Concatenate three numbers.""" + if _IS_PYDANTIC_V1 is not None: - a: int - b: int - c: int + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + """Concatenate three numbers.""" - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) + a: int + b: int + c: int - class ConcatModel(pydantic.BaseModel): - """Concatenate three numbers.""" + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) - a: int - b: int - c: int + class ConcatModel(pydantic.BaseModel): + """Concatenate three numbers.""" - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) + a: int + b: int + c: int + + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) - for class_or_function in [concat, Concat, ConcatDataclass, ConcatDataclassPydantic, ConcatModel]: + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: output_1 = tapify(class_or_function, command_line_args=["--a", "1", "--b", "2", "--c", "3"]) output_2 = tapify(class_or_function, command_line_args=["--a", "4", "--b", "5", "--c", "6"]) @@ -858,84 +984,54 @@ class ConcatDataclass: def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, self.c) - @pydantic.dataclasses.dataclass - class ConcatDataclassPydanticDocstring: - """Concatenate three numbers. - - :param a: The first number. - :param b: The second number. - :param c: The third number. - """ - - a: int - b: int - c: int - - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) - - @pydantic.dataclasses.dataclass - class ConcatDataclassPydanticFields: - """Concatenate three numbers. - - :param a: The first number. - """ - - # Mixing field types should be ok - a: int - b: int = pydantic.Field(description="The second number.") - c: int = pydantic.dataclasses.Field(description="The third number.") + if _IS_PYDANTIC_V1 is not None: - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) + @pydantic.dataclasses.dataclass + class ConcatDataclassPydantic: + """Concatenate three numbers. - class ConcatModelDocstring(pydantic.BaseModel): - """Concatenate three numbers. + :param a: The first number. + :param b: The second number. + :param c: The third number. + """ - :param a: The first number. - :param b: The second number. - :param c: The third number. - """ + a: int + b: int + c: int - a: int - b: int - c: int + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) + class ConcatModel(pydantic.BaseModel): + """Concatenate three numbers. - class ConcatModelFields(pydantic.BaseModel): - """Concatenate three numbers. + :param a: The first number. + :param b: The second number. + :param c: The third number. + """ - :param a: The first number. - """ + a: int + b: int + c: int - # Mixing field types should be ok - a: int - b: int = pydantic.dataclasses.Field(description="The second number.") - c: int = field(metadata={"description": "The third number."}) + def __eq__(self, other: str) -> bool: + return other == concat(self.a, self.b, self.c) - def __eq__(self, other: str) -> bool: - return other == concat(self.a, self.b, self.c) + pydantic_data_models = [ConcatDataclassPydantic, ConcatModel] + else: + pydantic_data_models = [] - for class_or_function in [ - concat, - Concat, - ConcatDataclass, - ConcatDataclassPydanticDocstring, - ConcatDataclassPydanticFields, - ConcatModelDocstring, - ConcatModelFields, - ]: + for class_or_function in [concat, Concat, ConcatDataclass] + pydantic_data_models: f = io.StringIO() with contextlib.redirect_stdout(f): with self.assertRaises(SystemExit): tapify(class_or_function, command_line_args=["-h"]) - self.assertIn("Concatenate three numbers.", f.getvalue()) - self.assertIn("--a A (int, required) The first number.", f.getvalue()) - self.assertIn("--b B (int, required) The second number.", f.getvalue()) - self.assertIn("--c C (int, required) The third number.", f.getvalue()) + stdout = f.getvalue() + self.assertIn("Concatenate three numbers.", stdout) + self.assertIn("--a A (int, required) The first number.", stdout) + self.assertIn("--b B (int, required) The second number.", stdout) + self.assertIn("--c C (int, required) The third number.", stdout) class TestTapifyExplicitBool(unittest.TestCase): diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 2f846e9..7470bd8 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,5 +1,5 @@ """ -Tests `tap.to_tap_class`. This test works for Pydantic v1 and v2. +Tests `tap.to_tap_class`. """ from contextlib import redirect_stdout, redirect_stderr import dataclasses @@ -8,13 +8,17 @@ import sys from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union -import pydantic import pytest from tap import to_tap_class, Tap -_IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" +try: + import pydantic +except ModuleNotFoundError: + _IS_PYDANTIC_V1 = None +else: + _IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" # To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 _IS_BEFORE_PY_310 = sys.version_info < (3, 10) @@ -72,58 +76,67 @@ def __init__(self, arg_int: int, arg_bool: bool = True, arg_list: Optional[List[ DataclassBuiltin = _Args -@_monkeypatch_eq -@pydantic.dataclasses.dataclass -class DataclassPydantic: - """ - Dataclass (pydantic) - """ +if _IS_PYDANTIC_V1 is None: + pass # will raise NameError if attempting to use DataclassPydantic or Model later +elif _IS_PYDANTIC_V1: + # For Pydantic v1 data models, we rely on the docstring to get descriptions - # Mixing field types should be ok - arg_int: int = pydantic.dataclasses.Field(description="some integer") - arg_bool: bool = pydantic.dataclasses.Field(default=True) - arg_list: Optional[List[str]] = pydantic.Field(default=None, description="some list of strings") + @_monkeypatch_eq + @pydantic.dataclasses.dataclass + class DataclassPydantic: + """ + Dataclass (pydantic v1) + :param arg_int: some integer + :param arg_list: some list of strings + """ -@_monkeypatch_eq -@pydantic.dataclasses.dataclass -class DataclassPydanticV1: # for Pydantic v1 data models, we rely on the docstring to get descriptions - """ - Dataclass (pydantic v1) + arg_int: int + arg_bool: bool = True + arg_list: Optional[List[str]] = None - :param arg_int: some integer - :param arg_list: some list of strings - """ + @_monkeypatch_eq + class Model(pydantic.BaseModel): + """ + Pydantic model (pydantic v1) - arg_int: int - arg_bool: bool = True - arg_list: Optional[List[str]] = None + :param arg_int: some integer + :param arg_list: some list of strings + """ + arg_int: int + arg_bool: bool = True + arg_list: Optional[List[str]] = None -@_monkeypatch_eq -class Model(pydantic.BaseModel): - """ - Pydantic model - """ +else: + # For pydantic v2 data models, we check the docstring and Field for the description - # Mixing field types should be ok - arg_int: int = pydantic.Field(description="some integer") - arg_bool: bool = pydantic.Field(default=True) - arg_list: Optional[List[str]] = pydantic.dataclasses.Field(default=None, description="some list of strings") + @_monkeypatch_eq + @pydantic.dataclasses.dataclass + class DataclassPydantic: + """ + Dataclass (pydantic) + :param arg_list: some list of strings + """ -@_monkeypatch_eq -class ModelV1(pydantic.BaseModel): # for Pydantic v1 data models, we rely on the docstring to get descriptions - """ - Pydantic model (pydantic v1) + # Mixing field types should be ok + arg_int: int = pydantic.dataclasses.Field(description="some integer") + arg_bool: bool = dataclasses.field(default=True) + arg_list: Optional[List[str]] = pydantic.Field(default=None) - :param arg_int: some integer - :param arg_list: some list of strings - """ + @_monkeypatch_eq + class Model(pydantic.BaseModel): + """ + Pydantic model - arg_int: int - arg_bool: bool = True - arg_list: Optional[List[str]] = None + :param arg_int: some integer + """ + + # Mixing field types should be ok + arg_int: int + arg_bool: bool = dataclasses.field(default=True) + arg_list: Optional[List[str]] = pydantic.dataclasses.Field(default=None, description="some list of strings") @pytest.fixture( @@ -135,10 +148,9 @@ class ModelV1(pydantic.BaseModel): # for Pydantic v1 data models, we rely on th DataclassBuiltin( 1, arg_bool=False, arg_list=["these", "values", "don't", "matter"] ), # to_tap_class also works on instances of data models. It ignores the attribute values - DataclassPydantic if not _IS_PYDANTIC_V1 else DataclassPydanticV1, - Model if not _IS_PYDANTIC_V1 else ModelV1, - # We can test instances of DataclassPydantic and Model for pydantic v2 but not v1 - ], + ] + + ([] if _IS_PYDANTIC_V1 is None else [DataclassPydantic, Model]), + # NOTE: instances of DataclassPydantic and Model can be tested for pydantic v2 but not v1 ) def data_model(request: pytest.FixtureRequest): """ From 878162afa4318cedc4055e89452ea071e3fad0dc Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 08:29:15 -0800 Subject: [PATCH 34/40] dont require pydantic for test workflow --- .github/workflows/tests.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b436e0c..60d40ff 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,13 +36,21 @@ jobs: git config --global user.name "Your Name" python -m pip install --upgrade pip python -m pip install flake8 pytest - python -m pip install -e ".[dev]" + python -m pip install -e . - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest + - name: Test without pydantic run: | pytest + - name: Test with pydantic v1 + run: | + python -m pip install "pydantic < 2" + pytest + - name: Test with pydantic v2 + run: | + python -m pip install "pydantic >= 2" + pytest From 48d237c7e2fd0840fc0aafbb000636244e3cb99f Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 08:38:04 -0800 Subject: [PATCH 35/40] unused field --- tests/test_tapify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_tapify.py b/tests/test_tapify.py index 7d6ac0d..3289663 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -2,7 +2,7 @@ Tests `tap.tapify`. Currently requires Pydantic v2. """ import contextlib -from dataclasses import dataclass, field +from dataclasses import dataclass import io import sys from typing import Dict, List, Optional, Tuple, Any From 0c439f66033ab74aa604d6aaa4274a4da6ca5fc6 Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 09:08:32 -0800 Subject: [PATCH 36/40] use type_to_str instead --- tests/test_to_tap_class.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 7470bd8..c541c0e 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -11,6 +11,7 @@ import pytest from tap import to_tap_class, Tap +from tap.utils import type_to_str try: @@ -346,7 +347,7 @@ def test_subclasser_simple_help_message(data_model: Any): --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) --arg_list [ARG_LIST {_ARG_LIST_DOTS}] - ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings + ({type_to_str(Optional[List[str]])}, default=None) some list of strings -h, --help show this help message and exit """.lstrip( "\n" @@ -425,7 +426,7 @@ def test_subclasser_complex_help_message(data_model: Any): --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) --arg_list [ARG_LIST {_ARG_LIST_DOTS}] - ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings + ({type_to_str(Optional[List[str]])}, default=None) some list of strings -h, --help show this help message and exit """.lstrip( "\n" @@ -516,7 +517,7 @@ def test_subclasser_subparser( --arg_int ARG_INT (int, required) some integer --arg_bool (bool, default=True) --arg_list [ARG_LIST {_ARG_LIST_DOTS}] - ({str(Optional[List[str]]).replace('typing.', '')}, default=None) some list of strings + ({type_to_str(Optional[List[str]])}, default=None) some list of strings -h, --help show this help message and exit """, ), From d83bc1d48067c51bcf99509ea3049ea4e9dc5ec6 Mon Sep 17 00:00:00 2001 From: kddubey Date: Tue, 23 Jan 2024 13:31:57 -0800 Subject: [PATCH 37/40] little things --- tap/tapify.py | 22 ++++++++++++---------- tests/test_to_tap_class.py | 10 +++++----- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/tap/tapify.py b/tap/tapify.py index 2d01fa5..5dce02e 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -73,14 +73,14 @@ class _TapData: "If true, ignore extra arguments and only parse known arguments" -def _is_pydantic_base_model(obj: Union[Any, Type[Any]]) -> bool: +def _is_pydantic_base_model(obj: Union[Type[Any], Any]) -> bool: if inspect.isclass(obj): # issublcass requires that obj is a class return issubclass(obj, BaseModel) else: return isinstance(obj, BaseModel) -def _is_pydantic_dataclass(obj: Union[Any, Type[Any]]) -> bool: +def _is_pydantic_dataclass(obj: Union[Type[Any], Any]) -> bool: if _IS_PYDANTIC_V1: # There's no public function in v1. This is a somewhat safe but linear check return dataclasses.is_dataclass(obj) and any(key.startswith("__pydantic") for key in obj.__dict__) @@ -97,7 +97,7 @@ def _tap_data_from_data_model( - Pydantic dataclass (class or instance) - Pydantic BaseModel (class or instance). - The advantage of this function over func:`_tap_data_from_class_or_function` is that field/argument descriptions are + The advantage of this function over :func:`_tap_data_from_class_or_function` is that field/argument descriptions are extracted, b/c this function look at the fields of the data model. Note @@ -171,6 +171,8 @@ def _tap_data_from_class_or_function( class_or_function: _ClassOrFunction, func_kwargs: Dict[str, Any], param_to_description: Dict[str, str] ) -> _TapData: """ + Extract data by inspecting the signature of `class_or_function`. + Note ---- Deletes redundant keys from `func_kwargs` @@ -215,7 +217,7 @@ def _tap_data_from_class_or_function( return _TapData(args_data, has_kwargs, known_only) -def _is_data_model(obj: Union[Any, Type[Any]]) -> bool: +def _is_data_model(obj: Union[Type[Any], Any]) -> bool: return dataclasses.is_dataclass(obj) or _is_pydantic_base_model(obj) @@ -230,17 +232,17 @@ def _docstring(class_or_function) -> Docstring: def _tap_data(class_or_function: _ClassOrFunction, param_to_description: Dict[str, str], func_kwargs) -> _TapData: """ - Controls how class:`_TapData` is extracted from `class_or_function`. + Controls how :class:`_TapData` is extracted from `class_or_function`. """ is_pydantic_v1_data_model = _IS_PYDANTIC_V1 and ( _is_pydantic_base_model(class_or_function) or _is_pydantic_dataclass(class_or_function) ) if _is_data_model(class_or_function) and not is_pydantic_v1_data_model: - # Data models from Pydantic v1 don't lend itself well to _tap_data_from_data_model. _tap_data_from_data_model - # looks at the data model's fields. In Pydantic v1, the field.type_ attribute stores the field's - # annotation/type. But (in Pydantic v1) there's a bug where field.type_ is set to the inner-most type of a - # subscripted type. For example, annotating a field with list[str] causes field.type_ to be str, not list[str]. - # To get around this, we'll extract _TapData by looking at the signature of the data model + # Data models from Pydantic v1 don't lend themselves well to _tap_data_from_data_model. + # _tap_data_from_data_model looks at the data model's fields. In Pydantic v1, the field.type_ attribute stores + # the field's annotation/type. But (in Pydantic v1) there's a bug where field.type_ is set to the inner-most + # type of a subscripted type. For example, annotating a field with list[str] causes field.type_ to be str, not + # list[str]. To get around this, we'll extract _TapData by looking at the signature of the data model return _tap_data_from_data_model(class_or_function, func_kwargs, param_to_description) # TODO: allow passing func_kwargs to a Pydantic BaseModel return _tap_data_from_class_or_function(class_or_function, func_kwargs, param_to_description) diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index c541c0e..1c86245 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -21,9 +21,9 @@ else: _IS_PYDANTIC_V1 = pydantic.__version__ < "2.0.0" + # To properly test the help message, we need to know how argparse formats it. It changed from 3.8 -> 3.9 -> 3.10 -_IS_BEFORE_PY_310 = sys.version_info < (3, 10) -_OPTIONS_TITLE = "options" if not _IS_BEFORE_PY_310 else "optional arguments" +_OPTIONS_TITLE = "options" if not sys.version_info < (3, 10) else "optional arguments" _ARG_LIST_DOTS = "..." if not sys.version_info < (3, 9) else "[ARG_LIST ...]" @@ -335,7 +335,7 @@ def test_subclasser_simple( _test_subclasser(subclasser_simple, data_model, args_string_and_arg_to_expected_value) -# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") +# @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") def test_subclasser_simple_help_message(data_model: Any): description = "Script description" help_message_expected = f""" @@ -412,7 +412,7 @@ def test_subclasser_complex( _test_subclasser(subclasser_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) -# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") +# @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") def test_subclasser_complex_help_message(data_model: Any): description = "Script description" help_message_expected = f""" @@ -494,7 +494,7 @@ def test_subclasser_subparser( _test_subclasser(subclasser_subparser, data_model, args_string_and_arg_to_expected_value, test_call=False) -# @pytest.mark.skipif(_IS_BEFORE_PY_310, reason="argparse is different. Need to fix help_message_expected") +# @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") @pytest.mark.parametrize( "args_string_and_description_and_expected_message", [ From 3523da2ffe0e267bcdeeba9545a5a6385facb97d Mon Sep 17 00:00:00 2001 From: kddubey Date: Thu, 1 Feb 2024 13:09:39 -0800 Subject: [PATCH 38/40] add general pattern --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 6d94900..3fa0863 100644 --- a/README.md +++ b/README.md @@ -717,7 +717,7 @@ Pydantic [Models](https://docs.pydantic.dev/latest/concepts/models/) and [dataclasses](https://docs.pydantic.dev/latest/concepts/dataclasses/) can be `tapify`d. ```python -# square_dataclass.py +# square_pydantic.py from pydantic import BaseModel, Field from tap import tapify @@ -841,12 +841,14 @@ Running `python person.py --name Jesse --age 1` prints `My name is Jesse.` follo ### Explicit boolean arguments Tapify supports explicit specification of boolean arguments (see [bool](#bool) for more details). By default, `explicit_bool=False` and it can be set with `tapify(..., explicit_bool=True)`. -## to_tap_class +## Convert to a `Tap` class `to_tap_class` turns a function or class into a `Tap` class. The returned class can be [subclassed](#subclassing) to add special argument behavior. For example, you can override [`configure`](#configuring-arguments) and -[`process_args`](#argument-processing). If the object can be `tapify`d, then it can be `to_tap_class`d, and vice-versa. -`to_tap_class` provides full control over argument parsing. +[`process_args`](#argument-processing). + +If the object can be `tapify`d, then it can be `to_tap_class`d, and vice-versa. `to_tap_class` provides full control +over argument parsing. ### Examples @@ -879,5 +881,12 @@ Running `python main.py --package tap` will print `Project instance: package='ta ### Complex +The general pattern is: + +```python +class MyCustomTap(to_tap_class(my_class_or_function)): + # Special argument behavior, e.g., override configure and/or process_args +``` + Please see `demo_data_model.py` for an example of overriding [`configure`](#configuring-arguments) and [`process_args`](#argument-processing). From 28e3c19427edbc674b5d8e489e0de13603e76678 Mon Sep 17 00:00:00 2001 From: kddubey Date: Sun, 18 Feb 2024 17:03:06 -0800 Subject: [PATCH 39/40] littler things --- README.md | 3 +++ tests/test_to_tap_class.py | 25 +++++++++++-------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3fa0863..410f93c 100644 --- a/README.md +++ b/README.md @@ -839,6 +839,7 @@ if __name__ == '__main__': Running `python person.py --name Jesse --age 1` prints `My name is Jesse.` followed by `My age is 1.`. Without `known_only=True`, the `tapify` calls would raise an error due to the extra argument. ### Explicit boolean arguments + Tapify supports explicit specification of boolean arguments (see [bool](#bool) for more details). By default, `explicit_bool=False` and it can be set with `tapify(..., explicit_bool=True)`. ## Convert to a `Tap` class @@ -884,6 +885,8 @@ Running `python main.py --package tap` will print `Project instance: package='ta The general pattern is: ```python +from tap import to_tap_class + class MyCustomTap(to_tap_class(my_class_or_function)): # Special argument behavior, e.g., override configure and/or process_args ``` diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 1c86245..2dfd59c 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -1,6 +1,7 @@ """ Tests `tap.to_tap_class`. """ + from contextlib import redirect_stdout, redirect_stderr import dataclasses import io @@ -255,28 +256,24 @@ def _test_subclasser( args_string, arg_to_expected_value = args_string_and_arg_to_expected_value TapSubclass = subclasser(class_or_function) assert issubclass(TapSubclass, Tap) - tap = TapSubclass(description="Script description") # description is a kwarg for argparse.ArgumentParser + tap = TapSubclass(description="Script description") - # args_string is an invalid argument combo if isinstance(arg_to_expected_value, SystemExit): - # We need to get the error message by reading stdout stderr = _test_raises_system_exit(tap, args_string) assert str(arg_to_expected_value) in stderr - return - if isinstance(arg_to_expected_value, BaseException): + elif isinstance(arg_to_expected_value, BaseException): expected_exception = arg_to_expected_value.__class__ expected_error_message = str(arg_to_expected_value) or None with pytest.raises(expected_exception=expected_exception, match=expected_error_message): args = tap.parse_args(args_string.split()) - return - - # args_string is a valid argument combo - # Test that parsing works correctly - args = tap.parse_args(args_string.split()) - assert arg_to_expected_value == args.as_dict() - if test_call and callable(class_or_function): - result = class_or_function(**args.as_dict()) - assert result == _Args(**arg_to_expected_value) + else: + # args_string is a valid argument combo + # Test that parsing works correctly + args = tap.parse_args(args_string.split()) + assert arg_to_expected_value == args.as_dict() + if test_call and callable(class_or_function): + result = class_or_function(**args.as_dict()) + assert result == _Args(**arg_to_expected_value) def _test_subclasser_message( From 7461ce772d24aadf8dd21821a242716fb67e3d87 Mon Sep 17 00:00:00 2001 From: kddubey Date: Wed, 28 Feb 2024 18:46:43 -0800 Subject: [PATCH 40/40] allow extra args for Pydantic BaseModels with extra=allow unflake flaky test add some comments --- README.md | 2 +- demo_data_model.py | 3 +- tap/tapify.py | 7 +++- tests/test_tapify.py | 65 ++++++++++++++++++++++++++------------ tests/test_to_tap_class.py | 28 ++++++++-------- 5 files changed, 66 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 410f93c..1ad6851 100644 --- a/README.md +++ b/README.md @@ -746,7 +746,7 @@ docstring is used. In the example below, the description is provided in the docs For Pydantic v1 models and dataclasses, the argument's description must be provided in the class docstring: ```python -# square_dataclass.py +# square_pydantic.py from pydantic import BaseModel from tap import tapify diff --git a/demo_data_model.py b/demo_data_model.py index 95925c7..6542d99 100644 --- a/demo_data_model.py +++ b/demo_data_model.py @@ -85,8 +85,7 @@ def process_args(self) -> None: # args = ModelTapWithSubparsing(description="Script description").parse_args() print("Parsed args:") print(args) - # Run the main function. Pydantic BaseModels ignore arguments which aren't one of their fields instead of raising an - # error + # Run the main function model = Model(**args.as_dict()) main(model) diff --git a/tap/tapify.py b/tap/tapify.py index 5dce02e..58f8c34 100644 --- a/tap/tapify.py +++ b/tap/tapify.py @@ -4,6 +4,7 @@ `to_tap_class`: convert a class or function into a `Tap` class, which can then be subclassed to add special argument handling """ + import dataclasses import inspect from typing import Any, Callable, Dict, List, Optional, Sequence, Type, TypeVar, Union @@ -121,6 +122,8 @@ def is_required(field: dataclasses.Field) -> bool: def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optional[Type] = None) -> _ArgData: annotation = field.annotation if annotation is None else annotation + # Prefer the description from param_to_description (from the data model / class docstring) over the + # field.description b/c a docstring can be modified on the fly w/o causing real issues description = param_to_description.get(name, field.description) return _ArgData(name, annotation, field.is_required(), field.default, description) @@ -131,7 +134,9 @@ def arg_data_from_pydantic(name: str, field: _PydanticField, annotation: Optiona known_only = False elif _is_pydantic_base_model(data_model): name_to_field = data_model.model_fields - is_extra_ok = data_model.model_config.get("extra", "ignore") != "forbid" + # For backwards compatibility, only allow new kwargs to get assigned if the model is explicitly configured to do + # so via extra="allow". See https://docs.pydantic.dev/latest/api/config/#pydantic.config.ConfigDict.extra + is_extra_ok = data_model.model_config.get("extra", "ignore") == "allow" has_kwargs = is_extra_ok known_only = is_extra_ok else: diff --git a/tests/test_tapify.py b/tests/test_tapify.py index 3289663..f2aa387 100644 --- a/tests/test_tapify.py +++ b/tests/test_tapify.py @@ -1,6 +1,7 @@ """ Tests `tap.tapify`. Currently requires Pydantic v2. """ + import contextlib from dataclasses import dataclass import io @@ -510,14 +511,6 @@ def __eq__(self, other: str) -> bool: return other == concat(self.so, self.few) class ConcatModel(pydantic.BaseModel): - if _IS_PYDANTIC_V1: - - class Config: - extra = pydantic.Extra.forbid # by default, pydantic ignores extra arguments - - else: - model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments - so: int few: float @@ -689,14 +682,6 @@ def __eq__(self, other: str) -> bool: return other == concat(self.i, self.like, self.k, self.w, self.args, self.always) class ConcatModel(pydantic.BaseModel): - if _IS_PYDANTIC_V1: - - class Config: - extra = pydantic.Extra.forbid # by default, pydantic ignores extra arguments - - else: - model_config = pydantic.ConfigDict(extra="forbid") # by default, pydantic ignores extra arguments - i: int like: float k: int @@ -1097,7 +1082,45 @@ def concat(a: int, b: int = 2, **kwargs) -> str: """ return f'{a}_{b}_{"-".join(f"{k}={v}" for k, v in kwargs.items())}' - self.concat_function = concat + if _IS_PYDANTIC_V1 is not None: + + class ConcatModel(pydantic.BaseModel): + """Concatenate three numbers. + + :param a: The first number. + :param b: The second number. + """ + + if _IS_PYDANTIC_V1: + + class Config: + extra = pydantic.Extra.allow # by default, pydantic ignores extra arguments + + else: + model_config = pydantic.ConfigDict(extra="allow") # by default, pydantic ignores extra arguments + + a: int + b: int = 2 + + def __eq__(self, other: str) -> bool: + if _IS_PYDANTIC_V1: + # Get the kwarg names in the correct order by parsing other + kwargs_str = other.split("_")[-1] + if not kwargs_str: + kwarg_names = [] + else: + kwarg_names = [kv_str.split("=")[0] for kv_str in kwargs_str.split("-")] + kwargs = {name: getattr(self, name) for name in kwarg_names} + # Need to explictly check that the extra names from other are identical to what's stored in self + # Checking other == concat(...) isn't sufficient b/c self could have more extra fields + assert set(kwarg_names) == set(self.__dict__.keys()) - set(self.__fields__.keys()) + else: + kwargs = self.model_extra + return other == concat(self.a, self.b, **kwargs) + + pydantic_data_models = [ConcatModel] + else: + pydantic_data_models = [] class Concat: def __init__(self, a: int, b: int = 2, **kwargs: Dict[str, str]): @@ -1113,22 +1136,22 @@ def __init__(self, a: int, b: int = 2, **kwargs: Dict[str, str]): def __eq__(self, other: str) -> bool: return other == concat(self.a, self.b, **self.kwargs) - self.concat_class = Concat + self.class_or_functions = [concat, Concat] + pydantic_data_models def test_tapify_empty_kwargs(self) -> None: - for class_or_function in [self.concat_function, self.concat_class]: + for class_or_function in self.class_or_functions: output = tapify(class_or_function, command_line_args=["--a", "1"]) self.assertEqual(output, "1_2_") def test_tapify_has_kwargs(self) -> None: - for class_or_function in [self.concat_function, self.concat_class]: + for class_or_function in self.class_or_functions: output = tapify(class_or_function, command_line_args=["--a", "1", "--c", "3", "--d", "4"]) self.assertEqual(output, "1_2_c=3-d=4") def test_tapify_has_kwargs_replace_default(self) -> None: - for class_or_function in [self.concat_function, self.concat_class]: + for class_or_function in self.class_or_functions: output = tapify(class_or_function, command_line_args=["--a", "1", "--c", "3", "--b", "5", "--d", "4"]) self.assertEqual(output, "1_5_c=3-d=4") diff --git a/tests/test_to_tap_class.py b/tests/test_to_tap_class.py index 2dfd59c..730e4ed 100644 --- a/tests/test_to_tap_class.py +++ b/tests/test_to_tap_class.py @@ -154,9 +154,9 @@ class Model(pydantic.BaseModel): + ([] if _IS_PYDANTIC_V1 is None else [DataclassPydantic, Model]), # NOTE: instances of DataclassPydantic and Model can be tested for pydantic v2 but not v1 ) -def data_model(request: pytest.FixtureRequest): +def class_or_function_(request: pytest.FixtureRequest): """ - Same meaning as class_or_function. Only difference is that data_model is parametrized. + Parametrized class_or_function. """ return request.param @@ -327,13 +327,13 @@ def replace_whitespace(string: str) -> str: ], ) def test_subclasser_simple( - data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] + class_or_function_: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] ): - _test_subclasser(subclasser_simple, data_model, args_string_and_arg_to_expected_value) + _test_subclasser(subclasser_simple, class_or_function_, args_string_and_arg_to_expected_value) # @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") -def test_subclasser_simple_help_message(data_model: Any): +def test_subclasser_simple_help_message(class_or_function_: Any): description = "Script description" help_message_expected = f""" usage: pytest --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] @@ -349,7 +349,7 @@ def test_subclasser_simple_help_message(data_model: Any): """.lstrip( "\n" ) - _test_subclasser_message(subclasser_simple, data_model, help_message_expected, description=description) + _test_subclasser_message(subclasser_simple, class_or_function_, help_message_expected, description=description) # Test subclasser_complex @@ -403,14 +403,14 @@ def test_subclasser_simple_help_message(data_model: Any): ], ) def test_subclasser_complex( - data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] + class_or_function_: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] ): # Currently setting test_call=False b/c all data models except the pydantic Model don't accept extra args - _test_subclasser(subclasser_complex, data_model, args_string_and_arg_to_expected_value, test_call=False) + _test_subclasser(subclasser_complex, class_or_function_, args_string_and_arg_to_expected_value, test_call=False) # @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") -def test_subclasser_complex_help_message(data_model: Any): +def test_subclasser_complex_help_message(class_or_function_: Any): description = "Script description" help_message_expected = f""" usage: pytest [-arg ARGUMENT_WITH_REALLY_LONG_NAME] --arg_int ARG_INT [--arg_bool] [--arg_list [ARG_LIST {_ARG_LIST_DOTS}]] [-h] @@ -428,7 +428,7 @@ def test_subclasser_complex_help_message(data_model: Any): """.lstrip( "\n" ) - _test_subclasser_message(subclasser_complex, data_model, help_message_expected, description=description) + _test_subclasser_message(subclasser_complex, class_or_function_, help_message_expected, description=description) # Test subclasser_subparser @@ -485,10 +485,10 @@ def test_subclasser_complex_help_message(data_model: Any): ], ) def test_subclasser_subparser( - data_model: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] + class_or_function_: Any, args_string_and_arg_to_expected_value: Tuple[str, Union[Dict[str, Any], BaseException]] ): # Currently setting test_call=False b/c all data models except the pydantic Model don't accept extra args - _test_subclasser(subclasser_subparser, data_model, args_string_and_arg_to_expected_value, test_call=False) + _test_subclasser(subclasser_subparser, class_or_function_, args_string_and_arg_to_expected_value, test_call=False) # @pytest.mark.skipif(sys.version_info < (3, 10), reason="argparse is different. Need to fix help_message_expected") @@ -545,9 +545,9 @@ def test_subclasser_subparser( ], ) def test_subclasser_subparser_help_message( - data_model: Any, args_string_and_description_and_expected_message: Tuple[str, str] + class_or_function_: Any, args_string_and_description_and_expected_message: Tuple[str, str] ): args_string, description, expected_message = args_string_and_description_and_expected_message _test_subclasser_message( - subclasser_subparser, data_model, expected_message, description=description, args_string=args_string + subclasser_subparser, class_or_function_, expected_message, description=description, args_string=args_string )