feature: dots in paths (#2008)

* sanitize dots in path names

* add flag that sanitizes dots in filenames as modules

* add missing __init__.py files when creating submodules.

* add test

* fix and extend tests

* remove unnecessary list comprehension

* remove unnecessary list comprehension

* fix set comprehension

* remove unrelated change

* extract inner function for testing

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* extract classmethods to try and fix tests

* Revert "extract classmethods to try and fix tests"

This reverts commit bdbfec7.

* Revert "extract inner function for testing"

This reverts commit 20b694e.

* write functional test

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* increase test coverage

* fix test

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Koudai Aono <koxudaxi@gmail.com>

datamodel_code_generator/__init__.py

@@ -302,6 +302,7 @@ def generate(
     custom_formatters_kwargs: Optional[Dict[str, Any]] = None,
     use_pendulum: bool = False,
     http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None,
+    treat_dots_as_module: bool = False,
     use_exact_imports: bool = False,
 ) -> None:
     remote_text_cache: DefaultPutDict[str, str] = DefaultPutDict()
@@ -463,6 +464,7 @@ def get_header_and_first_line(csv_file: IO[str]) -> Dict[str, Any]:
         custom_formatters_kwargs=custom_formatters_kwargs,
         use_pendulum=use_pendulum,
         http_query_parameters=http_query_parameters,
+        treat_dots_as_module=treat_dots_as_module,
         use_exact_imports=use_exact_imports,
         **kwargs,
     )

datamodel_code_generator/__main__.py

@@ -312,6 +312,7 @@ def validate_root(cls, values: Any) -> Any:
     custom_formatters_kwargs: Optional[TextIOBase] = None
     use_pendulum: bool = False
     http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None
+    treat_dot_as_module: bool = False
     use_exact_imports: bool = False
 
     def merge_args(self, args: Namespace) -> None:
@@ -509,6 +510,7 @@ def main(args: Optional[Sequence[str]] = None) -> Exit:
             custom_formatters_kwargs=custom_formatters_kwargs,
             use_pendulum=config.use_pendulum,
             http_query_parameters=config.http_query_parameters,
+            treat_dots_as_module=config.treat_dot_as_module,
             use_exact_imports=config.use_exact_imports,
         )
         return Exit.OK

datamodel_code_generator/arguments.py

@@ -160,6 +160,12 @@ def start_section(self, heading: Optional[str]) -> None:
     help='target python version (default: 3.7)',
     choices=[v.value for v in PythonVersion],
 )
+model_options.add_argument(
+    '--treat-dot-as-module',
+    help='treat dotted module names as modules',
+    action='store_true',
+    default=False,
+)
 model_options.add_argument(
     '--use-schema-description',
     help='Use schema description to populate class docstring',

datamodel_code_generator/parser/base.py

@@ -400,6 +400,7 @@ def __init__(
         custom_formatters_kwargs: Optional[Dict[str, Any]] = None,
         use_pendulum: bool = False,
         http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None,
+        treat_dots_as_module: bool = False,
         use_exact_imports: bool = False,
     ) -> None:
         self.data_type_manager: DataTypeManager = data_type_manager_type(
@@ -524,6 +525,7 @@ def __init__(
         self.known_third_party = known_third_party
         self.custom_formatter = custom_formatters
         self.custom_formatters_kwargs = custom_formatters_kwargs
+        self.treat_dots_as_module = treat_dots_as_module
 
     @property
     def iter_source(self) -> Iterator[Source]:
@@ -676,9 +678,8 @@ def __replace_duplicate_name_in_module(cls, models: List[DataModel]) -> None:
                 model.class_name = duplicate_name
                 model_names[duplicate_name] = model
 
-    @classmethod
     def __change_from_import(
-        cls,
+        self,
         models: List[DataModel],
         imports: Imports,
         scoped_model_resolver: ModelResolver,
@@ -715,6 +716,13 @@ def __change_from_import(
                             from_, import_, data_type.reference.short_name
                         )
                     import_ = import_.replace('-', '_')
+                    if (
+                        len(model.module_path) > 1
+                        and model.module_path[-1].count('.') > 0
+                        and not self.treat_dots_as_module
+                    ):
+                        rel_path_depth = model.module_path[-1].count('.')
+                        from_ = from_[rel_path_depth:]
 
                 alias = scoped_model_resolver.add(full_path, import_).name
 
@@ -1161,6 +1169,32 @@ def __set_one_literal_on_default(self, models: List[DataModel]) -> None:
                 if model_field.nullable is not True:  # pragma: no cover
                     model_field.nullable = False
 
+    @classmethod
+    def __postprocess_result_modules(cls, results):
+        def process(input_tuple) -> Tuple[str, ...]:
+            r = []
+            for item in input_tuple:
+                p = item.split('.')
+                if len(p) > 1:
+                    r.extend(p[:-1])
+                    r.append(p[-1])
+                else:
+                    r.append(item)
+
+            r = r[:-2] + [f'{r[-2]}.{r[-1]}']
+            return tuple(r)
+
+        results = {process(k): v for k, v in results.items()}
+
+        init_result = [v for k, v in results.items() if k[-1] == '__init__.py'][0]
+        folders = {t[:-1] if t[-1].endswith('.py') else t for t in results.keys()}
+        for folder in folders:
+            for i in range(len(folder)):
+                subfolder = folder[: i + 1]
+                init_file = subfolder + ('__init__.py',)
+                results.update({init_file: init_result})
+        return results
+
     def __change_imported_model_name(
         self,
         models: List[DataModel],
@@ -1338,4 +1372,20 @@ class Processed(NamedTuple):
         if [*results] == [('__init__.py',)]:
             return results[('__init__.py',)].body
 
+        results = {tuple(i.replace('-', '_') for i in k): v for k, v in results.items()}
+        results = (
+            self.__postprocess_result_modules(results)
+            if self.treat_dots_as_module
+            else {
+                tuple(
+                    (
+                        part[: part.rfind('.')].replace('.', '_')
+                        + part[part.rfind('.') :]
+                    )
+                    for part in k
+                ): v
+                for k, v in results.items()
+            }
+        )
+
         return results

datamodel_code_generator/parser/graphql.py

@@ -158,6 +158,7 @@ def __init__(
         custom_formatters_kwargs: Optional[Dict[str, Any]] = None,
         use_pendulum: bool = False,
         http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None,
+        treat_dots_as_module: bool = False,
         use_exact_imports: bool = False,
     ) -> None:
         super().__init__(
@@ -226,6 +227,7 @@ def __init__(
             custom_formatters_kwargs=custom_formatters_kwargs,
             use_pendulum=use_pendulum,
             http_query_parameters=http_query_parameters,
+            treat_dots_as_module=treat_dots_as_module,
             use_exact_imports=use_exact_imports,
         )
 

datamodel_code_generator/parser/jsonschema.py

@@ -440,6 +440,7 @@ def __init__(
         custom_formatters_kwargs: Optional[Dict[str, Any]] = None,
         use_pendulum: bool = False,
         http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None,
+        treat_dots_as_module: bool = False,
         use_exact_imports: bool = False,
     ) -> None:
         super().__init__(
@@ -508,6 +509,7 @@ def __init__(
             custom_formatters_kwargs=custom_formatters_kwargs,
             use_pendulum=use_pendulum,
             http_query_parameters=http_query_parameters,
+            treat_dots_as_module=treat_dots_as_module,
             use_exact_imports=use_exact_imports,
         )
 

datamodel_code_generator/parser/openapi.py

@@ -222,6 +222,7 @@ def __init__(
         custom_formatters_kwargs: Optional[Dict[str, Any]] = None,
         use_pendulum: bool = False,
         http_query_parameters: Optional[Sequence[Tuple[str, str]]] = None,
+        treat_dots_as_module: bool = False,
         use_exact_imports: bool = False,
     ):
         super().__init__(
@@ -290,6 +291,7 @@ def __init__(
             custom_formatters_kwargs=custom_formatters_kwargs,
             use_pendulum=use_pendulum,
             http_query_parameters=http_query_parameters,
+            treat_dots_as_module=treat_dots_as_module,
             use_exact_imports=use_exact_imports,
         )
         self.open_api_scopes: List[OpenAPIScope] = openapi_scopes or [

tests/data/expected/main/treat_dot_as_module/complex/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  treat_dot_as_module
+#   timestamp: 2019-07-26T00:00:00+00:00

tests/data/expected/main/treat_dot_as_module/complex/directory/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  treat_dot_as_module
+#   timestamp: 2019-07-26T00:00:00+00:00

tests/data/expected/main/treat_dot_as_module/complex/directory/api/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  treat_dot_as_module
+#   timestamp: 2019-07-26T00:00:00+00:00

tests/data/expected/main/treat_dot_as_module/complex/directory/api/path/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  treat_dot_as_module
+#   timestamp: 2019-07-26T00:00:00+00:00

tests/data/expected/main/treat_dot_as_module/complex/directory/api/path/input.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/api.path.input.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+from ... import schema
+
+
+class Input(BaseModel):
+    input: Optional[Any] = Field('input', title='Input')
+    extType: Optional[schema.ExtType] = None

tests/data/expected/main/treat_dot_as_module/complex/directory/api/path/output.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/api.path.output.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class Output(BaseModel):
+    output: Optional[Any] = Field('output', title='Output')

tests/data/expected/main/treat_dot_as_module/complex/directory/schema.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/schema.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ExtType(BaseModel):
+    ExtType: Optional[Any] = Field(None, title='ExtType')

tests/data/expected/main/treat_dot_not_as_module/complex.directory/__init__.py

@@ -0,0 +1,3 @@
+# generated by datamodel-codegen:
+#   filename:  treat_dot_as_module
+#   timestamp: 2019-07-26T00:00:00+00:00

tests/data/expected/main/treat_dot_not_as_module/complex.directory/api_path_input.py

@@ -0,0 +1,16 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/api.path.input.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+from . import schema
+
+
+class Input(BaseModel):
+    input: Optional[Any] = Field('input', title='Input')
+    extType: Optional[schema.ExtType] = None

tests/data/expected/main/treat_dot_not_as_module/complex.directory/api_path_output.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/api.path.output.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class Output(BaseModel):
+    output: Optional[Any] = Field('output', title='Output')

tests/data/expected/main/treat_dot_not_as_module/complex.directory/schema.py

@@ -0,0 +1,13 @@
+# generated by datamodel-codegen:
+#   filename:  complex.directory/schema.json
+#   timestamp: 2019-07-26T00:00:00+00:00
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+class ExtType(BaseModel):
+    ExtType: Optional[Any] = Field(None, title='ExtType')

tests/data/jsonschema/treat_dot_as_module/complex.directory/api.path.input.json

@@ -0,0 +1,13 @@
+{
+  "properties": {
+    "input": {
+      "default": "input",
+      "title": "Input"
+    },
+    "extType": {
+      "$ref": "schema.json"
+    }
+  },
+  "title": "Input",
+  "type": "object"
+}

tests/data/jsonschema/treat_dot_as_module/complex.directory/api.path.output.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "output": {
+      "default": "output",
+      "title": "Output"
+    }
+  },
+  "title": "Output",
+  "type": "object"
+}

tests/data/jsonschema/treat_dot_as_module/complex.directory/schema.json

@@ -0,0 +1,10 @@
+{
+  "properties": {
+    "ExtType": {
+      "type": "a",
+      "title": "ExtType"
+    }
+  },
+  "title": "ExtType",
+  "type": "object"
+}

tests/parser/test_base.py

@@ -250,3 +250,36 @@ def test_no_additional_imports():
         source='',
     )
     assert len(new_parser.imports) == 0
+
+
+@pytest.mark.parametrize(
+    'input_data, expected',
+    [
+        (
+            {
+                ('folder1', 'module1.py'): 'content1',
+                ('folder1', 'module2.py'): 'content2',
+                ('folder1', '__init__.py'): 'init_content',
+            },
+            {
+                ('folder1', 'module1.py'): 'content1',
+                ('folder1', 'module2.py'): 'content2',
+                ('folder1', '__init__.py'): 'init_content',
+            },
+        ),
+        (
+            {
+                ('folder1.module', 'file.py'): 'content1',
+                ('folder1.module', '__init__.py'): 'init_content',
+            },
+            {
+                ('folder1', 'module', 'file.py'): 'content1',
+                ('folder1', '__init__.py'): 'init_content',
+                ('folder1', 'module', '__init__.py'): 'init_content',
+            },
+        ),
+    ],
+)
+def test_postprocess_result_modules(input_data, expected):
+    result = Parser._Parser__postprocess_result_modules(input_data)
+    assert result == expected