python: adjust basic typing information (OpenAPITools#17511)

* python: adjust basic typing information

This is an initial pass to fix and adjust the typing information for the
generated client. This is enough to have mypy runnning without complains
on all the (modern) generated clients (Pydantic v1 code is not checked
for instance)

mypy is also now run directly in the CI, so further changes will also be
checked and thus, will need to be compliant with good typing
information.

Note that this doesn't *fully* type all the code: mypy is not run in
"strict" mode and there are still many functions/methods/attributes
which are still not fully typed, but it's a first good step in that
direction.

* ApiResponse's raw_data can't be None

* Fix indentation

* Revert test changes

* run mypy on tests/ directory

* don't forcefully convert the client response headers to dict

* override petstore ApiResponse model

* adjust type of 'any/one_of_schemas' fields

.github/workflows/samples-python-client-echo-api.yaml

@@ -42,3 +42,7 @@ jobs:
       - name: Test
         working-directory: ${{ matrix.sample }}
         run: python -m pytest
+
+      - name: mypy
+        working-directory: ${{ matrix.sample }}
+        run: python -m mypy

.github/workflows/samples-python-petstore.yaml

@@ -57,3 +57,7 @@ jobs:
       - name: Test
         working-directory: ${{ matrix.sample }}
         run: poetry run pytest -v
+
+      - name: mypy
+        working-directory: ${{ matrix.sample }}
+        run: poetry run mypy

bin/configs/python-aiohttp.yaml

@@ -6,3 +6,9 @@ library: asyncio
 additionalProperties:
   packageName: petstore_api
   mapNumberTo: float
+nameMappings:
+  _type: underscore_type
+  type_: type_with_underscore
+modelNameMappings:
+  # The OpenAPI spec ApiResponse conflicts with the internal ApiResponse
+  ApiResponse: ModelApiResponse

bin/configs/python.yaml

@@ -10,3 +10,6 @@ additionalProperties:
 nameMappings:
   _type: underscore_type
   type_: type_with_underscore
+modelNameMappings:
+  # The OpenAPI spec ApiResponse conflicts with the internal ApiResponse
+  ApiResponse: ModelApiResponse

modules/openapi-generator/src/main/resources/python/api.mustache

@@ -4,11 +4,7 @@
 import warnings
 from pydantic import validate_call, Field, StrictFloat, StrictStr, StrictInt
 from typing import Any, Dict, List, Optional, Tuple, Union
-
-try:
-    from typing import Annotated
-except ImportError:
-    from typing_extensions import Annotated
+from typing_extensions import Annotated
 
 {{#imports}}
 {{import}}

modules/openapi-generator/src/main/resources/python/api_client.mustache

@@ -280,15 +280,13 @@ class ApiClient:
             )
 
         except ApiException as e:
-            if e.body:
-                e.body = e.body.decode('utf-8')
             raise e
 
         return response_data
 
     def response_deserialize(
         self,
-        response_data: rest.RESTResponse = None,
+        response_data: rest.RESTResponse,
         response_types_map=None
     ) -> ApiResponse:
         """Deserializes response into an object.
@@ -297,6 +295,8 @@ class ApiClient:
         :return: ApiResponse
         """
 
+        msg = "RESTResponse.read() must be called before passing it to response_deserialize()"
+        assert response_data.data is not None, msg
 
         response_type = response_types_map.get(str(response_data.status), None)
         if not response_type and isinstance(response_data.status, int) and 100 <= response_data.status <= 599:

modules/openapi-generator/src/main/resources/python/api_response.mustache

@@ -1,8 +1,8 @@
 """API response object."""
 
 from __future__ import annotations
-from typing import Dict, Optional, Generic, TypeVar
-from pydantic import Field, StrictInt, StrictStr, StrictBytes, BaseModel
+from typing import Optional, Generic, Mapping, TypeVar
+from pydantic import Field, StrictInt, StrictBytes, BaseModel
 
 T = TypeVar("T")
 
@@ -12,7 +12,7 @@ class ApiResponse(BaseModel, Generic[T]):
     """
 
     status_code: StrictInt = Field(description="HTTP status code")
-    headers: Optional[Dict[StrictStr, StrictStr]] = Field(None, description="HTTP headers")
+    headers: Optional[Mapping[str, str]] = Field(None, description="HTTP headers")
     data: T = Field(description="Deserialized data given the data type")
     raw_data: StrictBytes = Field(description="Raw data (HTTP response body)")
 

modules/openapi-generator/src/main/resources/python/asyncio/rest.mustache

@@ -6,6 +6,7 @@ import io
 import json
 import re
 import ssl
+from typing import Optional
 
 import aiohttp
 import aiohttp_retry
@@ -72,6 +73,7 @@ class RESTClientObject:
         )
 
         retries = configuration.retries
+        self.retry_client: Optional[aiohttp_retry.RetryClient]
         if retries is not None:
             self.retry_client = aiohttp_retry.RetryClient(
                 client_session=self.pool_manager,

modules/openapi-generator/src/main/resources/python/exceptions.mustache

@@ -2,7 +2,6 @@
 
 {{>partial_header}}
 from typing import Any, Optional
-
 from typing_extensions import Self
 
 class OpenApiException(Exception):

modules/openapi-generator/src/main/resources/python/model_anyof.mustache

@@ -11,6 +11,7 @@ import re  # noqa: F401
 {{/vendorExtensions.x-py-model-imports}}
 from typing import Union, Any, List, TYPE_CHECKING, Optional, Dict
 from typing_extensions import Literal, Self
+from pydantic import Field
 
 {{#lambda.uppercase}}{{{classname}}}{{/lambda.uppercase}}_ANY_OF_SCHEMAS = [{{#anyOf}}"{{.}}"{{^-last}}, {{/-last}}{{/anyOf}}]
 
@@ -27,7 +28,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         actual_instance: Optional[Union[{{#anyOf}}{{{.}}}{{^-last}}, {{/-last}}{{/anyOf}}]] = None
     else:
         actual_instance: Any = None
-    any_of_schemas: List[str] = Literal[{{#lambda.uppercase}}{{{classname}}}{{/lambda.uppercase}}_ANY_OF_SCHEMAS]
+    any_of_schemas: List[str] = Field(default=Literal[{{#anyOf}}"{{.}}"{{^-last}}, {{/-last}}{{/anyOf}}])
 
     model_config = {
         "validate_assignment": True,
@@ -153,22 +154,20 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         if self.actual_instance is None:
             return "null"
 
-        to_json = getattr(self.actual_instance, "to_json", None)
-        if callable(to_json):
+        if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json):
             return self.actual_instance.to_json()
         else:
             return json.dumps(self.actual_instance)
 
-    def to_dict(self) -> Dict:
+    def to_dict(self) -> Optional[Union[Dict, {{#anyOf}}{{.}}{{^-last}}, {{/-last}}{{/anyOf}}]]:
         """Returns the dict representation of the actual instance"""
         if self.actual_instance is None:
-            return "null"
+            return None
 
-        to_json = getattr(self.actual_instance, "to_json", None)
-        if callable(to_json):
+        if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict):
             return self.actual_instance.to_dict()
         else:
-            return json.dumps(self.actual_instance)
+            return self.actual_instance
 
     def to_str(self) -> str:
         """Returns the string representation of the actual instance"""

modules/openapi-generator/src/main/resources/python/model_generic.mustache

@@ -9,10 +9,8 @@ import json
 {{#vendorExtensions.x-py-model-imports}}
 {{{.}}}
 {{/vendorExtensions.x-py-model-imports}}
-try:
-    from typing import Self
-except ImportError:
-    from typing_extensions import Self
+from typing import Optional, Set
+from typing_extensions import Self
 
 class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}):
     """
@@ -87,15 +85,15 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
 {{#hasChildren}}
 {{#discriminator}}
     # JSON field name that stores the object type
-    __discriminator_property_name: ClassVar[List[str]] = '{{discriminator.propertyBaseName}}'
+    __discriminator_property_name: ClassVar[str] = '{{discriminator.propertyBaseName}}'
 
     # discriminator mappings
     __discriminator_value_class_map: ClassVar[Dict[str, str]] = {
         {{#mappedModels}}'{{{mappingName}}}': '{{{modelName}}}'{{^-last}},{{/-last}}{{/mappedModels}}
     }
 
     @classmethod
-    def get_discriminator_value(cls, obj: Dict) -> str:
+    def get_discriminator_value(cls, obj: Dict) -> Optional[str]:
         """Returns the discriminator value (object type) of the data"""
         discriminator_value = obj[cls.__discriminator_property_name]
         if discriminator_value:
@@ -115,7 +113,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         return json.dumps(self.to_dict())
 
     @classmethod
-    def from_json(cls, json_str: str) -> {{^hasChildren}}Self{{/hasChildren}}{{#hasChildren}}{{#discriminator}}Union[{{#children}}Self{{^-last}}, {{/-last}}{{/children}}]{{/discriminator}}{{^discriminator}}Self{{/discriminator}}{{/hasChildren}}:
+    def from_json(cls, json_str: str) -> Optional[{{^hasChildren}}Self{{/hasChildren}}{{#hasChildren}}{{#discriminator}}Union[{{#children}}Self{{^-last}}, {{/-last}}{{/children}}]{{/discriminator}}{{^discriminator}}Self{{/discriminator}}{{/hasChildren}}]:
         """Create an instance of {{{classname}}} from a JSON string"""
         return cls.from_dict(json.loads(json_str))
 
@@ -135,16 +133,18 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         * Fields in `self.additional_properties` are added to the output dict.
         {{/isAdditionalPropertiesTrue}}
         """
+        excluded_fields: Set[str] = set([
+            {{#vendorExtensions.x-py-readonly}}
+            "{{{.}}}",
+            {{/vendorExtensions.x-py-readonly}}
+            {{#isAdditionalPropertiesTrue}}
+            "additional_properties",
+            {{/isAdditionalPropertiesTrue}}
+        ])
+
         _dict = self.model_dump(
             by_alias=True,
-            exclude={
-                {{#vendorExtensions.x-py-readonly}}
-                "{{{.}}}",
-                {{/vendorExtensions.x-py-readonly}}
-                {{#isAdditionalPropertiesTrue}}
-                "additional_properties",
-                {{/isAdditionalPropertiesTrue}}
-            },
+            exclude=excluded_fields,
             exclude_none=True,
         )
         {{#allVars}}
@@ -234,10 +234,10 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         {{/allVars}}
         return _dict
 
+    {{#hasChildren}}
     @classmethod
-    def from_dict(cls, obj: Dict) -> {{^hasChildren}}Self{{/hasChildren}}{{#hasChildren}}{{#discriminator}}Union[{{#children}}Self{{^-last}}, {{/-last}}{{/children}}]{{/discriminator}}{{^discriminator}}Self{{/discriminator}}{{/hasChildren}}:
+    def from_dict(cls, obj: Dict) -> Optional[{{#discriminator}}Union[{{#children}}Self{{^-last}}, {{/-last}}{{/children}}]{{/discriminator}}{{^discriminator}}Self{{/discriminator}}]:
         """Create an instance of {{{classname}}} from a dict"""
-        {{#hasChildren}}
         {{#discriminator}}
         # look up the object type based on discriminator mapping
         object_type = cls.get_discriminator_value(obj)
@@ -249,8 +249,11 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
                              json.dumps(obj) + ". Discriminator property name: " + cls.__discriminator_property_name +
                              ", mapping: " + json.dumps(cls.__discriminator_value_class_map))
         {{/discriminator}}
-        {{/hasChildren}}
-        {{^hasChildren}}
+    {{/hasChildren}}
+    {{^hasChildren}}
+    @classmethod
+    def from_dict(cls, obj: Optional[Dict]) -> Optional[Self]:
+        """Create an instance of {{{classname}}} from a dict"""
         if obj is None:
             return None
 
@@ -277,7 +280,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
             {{^items.items.isPrimitiveType}}
             "{{{baseName}}}": [
                     [{{{items.items.dataType}}}.from_dict(_inner_item) for _inner_item in _item]
-                    for _item in obj.get("{{{baseName}}}")
+                    for _item in obj["{{{baseName}}}"]
                 ] if obj.get("{{{baseName}}}") is not None else None{{^-last}},{{/-last}}
             {{/items.items.isPrimitiveType}}
             {{/items.isArray}}
@@ -287,7 +290,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
             "{{{baseName}}}": obj.get("{{{baseName}}}"){{^-last}},{{/-last}}
             {{/items.isEnumOrRef}}
             {{^items.isEnumOrRef}}
-            "{{{baseName}}}": [{{{items.dataType}}}.from_dict(_item) for _item in obj.get("{{{baseName}}}")] if obj.get("{{{baseName}}}") is not None else None{{^-last}},{{/-last}}
+            "{{{baseName}}}": [{{{items.dataType}}}.from_dict(_item) for _item in obj["{{{baseName}}}"]] if obj.get("{{{baseName}}}") is not None else None{{^-last}},{{/-last}}
             {{/items.isEnumOrRef}}
             {{/items.isPrimitiveType}}
             {{#items.isPrimitiveType}}
@@ -320,14 +323,14 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
                         if _v is not None
                         else None
                 )
-                for _k, _v in obj.get("{{{baseName}}}").items()
+                for _k, _v in obj.get("{{{baseName}}}", {}).items()
             ){{^-last}},{{/-last}}
             {{/items.isArray}}
             {{/items.isContainer}}
             {{^items.isContainer}}
             "{{{baseName}}}": dict(
                 (_k, {{{items.dataType}}}.from_dict(_v))
-                for _k, _v in obj.get("{{{baseName}}}").items()
+                for _k, _v in obj["{{{baseName}}}"].items()
             )
             if obj.get("{{{baseName}}}") is not None
             else None{{^-last}},{{/-last}}
@@ -345,7 +348,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
             {{^isContainer}}
             {{^isPrimitiveType}}
             {{^isEnumOrRef}}
-            "{{{baseName}}}": {{{dataType}}}.from_dict(obj.get("{{{baseName}}}")) if obj.get("{{{baseName}}}") is not None else None{{^-last}},{{/-last}}
+            "{{{baseName}}}": {{{dataType}}}.from_dict(obj["{{{baseName}}}"]) if obj.get("{{{baseName}}}") is not None else None{{^-last}},{{/-last}}
             {{/isEnumOrRef}}
             {{#isEnumOrRef}}
             "{{{baseName}}}": obj.get("{{{baseName}}}"){{^-last}},{{/-last}}
@@ -370,7 +373,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
 
         {{/isAdditionalPropertiesTrue}}
         return _obj
-        {{/hasChildren}}
+    {{/hasChildren}}
 
 {{#vendorExtensions.x-py-postponed-model-imports.size}}
 {{#vendorExtensions.x-py-postponed-model-imports}}

modules/openapi-generator/src/main/resources/python/model_oneof.mustache

@@ -22,7 +22,7 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
     {{vendorExtensions.x-py-name}}: {{{vendorExtensions.x-py-typing}}}
 {{/composedSchemas.oneOf}}
     actual_instance: Optional[Union[{{#oneOf}}{{{.}}}{{^-last}}, {{/-last}}{{/oneOf}}]] = None
-    one_of_schemas: List[str] = Literal[{{#oneOf}}"{{.}}"{{^-last}}, {{/-last}}{{/oneOf}}]
+    one_of_schemas: List[str] = Field(default=Literal[{{#oneOf}}"{{.}}"{{^-last}}, {{/-last}}{{/oneOf}}])
 
     model_config = {
         "validate_assignment": True,
@@ -175,19 +175,17 @@ class {{classname}}({{#parent}}{{{.}}}{{/parent}}{{^parent}}BaseModel{{/parent}}
         if self.actual_instance is None:
             return "null"
 
-        to_json = getattr(self.actual_instance, "to_json", None)
-        if callable(to_json):
+        if hasattr(self.actual_instance, "to_json") and callable(self.actual_instance.to_json):
             return self.actual_instance.to_json()
         else:
             return json.dumps(self.actual_instance)
 
-    def to_dict(self) -> Dict:
+    def to_dict(self) -> Optional[Union[Dict, {{#oneOf}}{{{.}}}{{^-last}}, {{/-last}}{{/oneOf}}]]:
         """Returns the dict representation of the actual instance"""
         if self.actual_instance is None:
             return None
 
-        to_dict = getattr(self.actual_instance, "to_dict", None)
-        if callable(to_dict):
+        if hasattr(self.actual_instance, "to_dict") and callable(self.actual_instance.to_dict):
             return self.actual_instance.to_dict()
         else:
             # primitive type

modules/openapi-generator/src/main/resources/python/pyproject.mustache

@@ -32,10 +32,20 @@ typing-extensions = ">=4.7.1"
 pytest = ">=7.2.1"
 tox = ">=3.9.0"
 flake8 = ">=4.0.0"
+types-python-dateutil = ">=2.8.19.14"
+mypy = "1.4.1"
+
 
 [build-system]
 requires = ["setuptools"]
 build-backend = "setuptools.build_meta"
 
 [tool.pylint.'MESSAGES CONTROL']
 extension-pkg-whitelist = "pydantic"
+
+[tool.mypy]
+files = [
+  "{{{packageName}}}",
+  #"test",  # auto-generated tests
+  "tests", # hand-written tests
+]