(feat, python): upgrade python generator to pydantic v2 (#4197)

fern-api · Aug 5, 2024 · 8d7d799 · 8d7d799
1 parent c4bb11e
commit 8d7d799
Show file tree

Hide file tree

Showing 214 changed files with 476 additions and 494 deletions.
diff --git a/.github/workflows/python-generator.yml b/.github/workflows/python-generator.yml
@@ -65,14 +65,10 @@ jobs:
 
       - name: Install Dependencies - Pydantic V2
         working-directory: ./generators/python
-        run: |
-          poetry remove fern-fern-generator-exec-sdk
-          poetry add pydantic=^2.8
+        run: | 
+          poetry add pydantic=^2.8.2
           poetry lock
           poetry install
       - name: Unit Test - Pydantic V2
         working-directory: ./generators/python
-        # Run specific tests:
-        #   Some tests pull in dependencies that have older versions of internal SDKs installed, which are
-        #   incompatible with Pydantic V2 outright (specifically the use of __root__ in unions with utils).
-        run: poetry run pytest -sv ./tests/utils/ --ignore ./tests/utils/casing
+        run: poetry run pytest -sv
diff --git a/generators/python/core_utilities/sdk/jsonable_encoder.py b/generators/python/core_utilities/sdk/jsonable_encoder.py
@@ -50,7 +50,7 @@ def jsonable_encoder(obj: Any, custom_encoder: Optional[Dict[Any, Callable[[Any]
             obj_dict = obj_dict["root"]
         return jsonable_encoder(obj_dict, custom_encoder=encoder)
     if dataclasses.is_dataclass(obj):
-        obj_dict = dataclasses.asdict(obj)
+        obj_dict = dataclasses.asdict(obj)  # type: ignore
         return jsonable_encoder(obj_dict, custom_encoder=custom_encoder)
     if isinstance(obj, bytes):
         return base64.b64encode(obj).decode("utf-8")

diff --git a/generators/python/core_utilities/sdk/pydantic_utilities.py b/generators/python/core_utilities/sdk/pydantic_utilities.py
@@ -158,7 +158,7 @@ def validate(*args: typing.Any, **kwargs: typing.Any) -> AnyCallable:
             else:
                 wrapped_func = pydantic.root_validator(pre=pre)(func)  # type: ignore # Pydantic v1
 
-            return wrapped_func(*args, **kwargs)
+            return wrapped_func(*args, **kwargs)  # type: ignore # Pydantic v2
 
         return validate
 
@@ -172,7 +172,7 @@ def validate(*args: typing.Any, **kwargs: typing.Any) -> AnyCallable:
             if IS_PYDANTIC_V2:
                 wrapped_func = pydantic.field_validator(field_name, mode="before" if pre else "after")(func)  # type: ignore # Pydantic v2
             else:
-                wrapped_func = pydantic.validator(field_name, pre=pre)(func)
+                wrapped_func = pydantic.validator(field_name, pre=pre)(func)  # type: ignore # Pydantic v1
 
             return wrapped_func(*args, **kwargs)
 

diff --git a/generators/python/core_utilities/sdk/unchecked_base_model.py b/generators/python/core_utilities/sdk/unchecked_base_model.py
@@ -265,7 +265,7 @@ def _get_model_fields(model: typing.Type["Model"]) -> typing.Mapping[str, Pydant
     if IS_PYDANTIC_V2:
         return model.model_fields  # type: ignore # Pydantic v2
     else:
-        return model.__fields__
+        return model.__fields__  # type: ignore # Pydantic v1
 
 
 def _get_field_default(field: PydanticField) -> typing.Any:

diff --git a/generators/python/fastapi/CHANGELOG.md b/generators/python/fastapi/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.3.0] - 2024-08-04
+
+- Internal: The generator has now been upgraded to use Pydantic V2 internally. Note that 
+  there is no change to the generated code, however by leveraging Pydantic V2 you should notice
+  an improvement in `fern generate` times. 
+
 ## [1.2.0] - 2024-07-31
 
 - Feature: The FastAPI `async_handlers` configuration now accepts a list of string or a boolean so that users

diff --git a/generators/python/fastapi/VERSION b/generators/python/fastapi/VERSION
@@ -1 +1 @@
-1.2.0
+1.3.0
diff --git a/generators/python/poetry.lock b/generators/python/poetry.lock
diff --git a/generators/python/pydantic/CHANGELOG.md b/generators/python/pydantic/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.0] - 2024-08-04
+
+- Internal: The generator has now been upgraded to use Pydantic V2 internally. Note that 
+  there is no change to the generated code, however by leveraging Pydantic V2 you should notice
+  an improvement in `fern generate` times. 
+
 ## [1.1.0-rc0] - 2024-07-31
 
 - Internal: The generator now consumes IRv53.

diff --git a/generators/python/pydantic/VERSION b/generators/python/pydantic/VERSION
@@ -1 +1 @@
-1.1.0-rc0
+1.2.0
diff --git a/generators/python/pyproject.toml b/generators/python/pyproject.toml
@@ -6,19 +6,19 @@ authors = []
 
 [tool.poetry.dependencies]
 python = "^3.9"
-pydantic = "^1.9.2,<=1.10.14"
+pydantic = "^2.8.2"
 typer = {extras = ["all"], version = "^0.6.1"}
-fern-fern-generator-exec-sdk = {version = "0.0.899", source = "fern-prod"}
+fern-fern-generator-exec-sdk = {version = "0.82.5", source = "fern-prod"}
 ordered-set = "^4.1.0"
 pydantic-core = "^2.18.2"
-fern-fern-fdr-sdk = {version = "0.0.5525", source = "fern-prod"}
-fern-fern-generator-cli-sdk = {version = "0.0.39", source = "fern-prod"}
-fern_fern_ir_v53 = "53.1.0"
+fern-fern-fdr-sdk = {version = "0.98.20", source = "fern-prod"}
+fern-fern-generator-cli-sdk = {version = "0.0.59", source = "fern-prod"}
+fern_fern_ir_v53 = "53.2.5"
 
 [tool.poetry.dev-dependencies]
 pytest = "^7.4.2"
 typing_extensions = ">=4.10.0,<5"
-mypy = "0.981"
+mypy = "^1.11.1"
 black = "^23.9.1"
 flake8 = "^5.0.4"
 isort = "^5.10.1"

diff --git a/generators/python/sdk/CHANGELOG.md b/generators/python/sdk/CHANGELOG.md
@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.4.0] - 2024-08-02
+
+- Internal: The SDK generator has now been upgraded to use Pydantic V2 internally. Note that 
+  there is no change to the generated code, however by leveraging Pydantic V2 you should notice
+  an improvement in `fern generate` times. 
+
 ## [3.3.4] - 2024-08-02
 
 - Improvement: Aliased literals are also defaulted within Pydantic models, whereas previously only direct literals were defaulted.

diff --git a/generators/python/sdk/VERSION b/generators/python/sdk/VERSION
@@ -1 +1 @@
-3.3.4
+3.4.0
diff --git a/generators/python/src/fern_python/cli/abstract_generator.py b/generators/python/src/fern_python/cli/abstract_generator.py
@@ -5,8 +5,8 @@
 from typing import Literal, Optional, Sequence, Tuple, cast
 
 import fern.ir.resources as ir_types
-from fern.generator_exec.resources import GeneratorConfig, PypiMetadata
-from fern.generator_exec.resources.config import (
+from fern.generator_exec import GeneratorConfig, PypiMetadata
+from fern.generator_exec.config import (
     GeneratorPublishConfig,
     GithubOutputMode,
     PypiGithubPublishInfo,

diff --git a/generators/python/src/fern_python/cli/generator_cli.py b/generators/python/src/fern_python/cli/generator_cli.py
@@ -1,8 +1,8 @@
 import sys
 
 import fern.ir.resources as ir_types
-from fern.generator_exec.resources.config import GeneratorConfig
-from fern.generator_exec.resources.logging import (
+from fern.generator_exec.config import GeneratorConfig
+from fern.generator_exec.logging import (
     ErrorExitStatusUpdate,
     ExitStatusUpdate,
     GeneratorUpdate,

diff --git a/generators/python/src/fern_python/cli/publisher.py b/generators/python/src/fern_python/cli/publisher.py
@@ -1,8 +1,8 @@
 import subprocess
 from typing import List
 
-from fern.generator_exec.resources import logging
-from fern.generator_exec.resources.config import GeneratorConfig, GeneratorPublishConfig
+from fern.generator_exec import logging
+from fern.generator_exec.config import GeneratorConfig, GeneratorPublishConfig
 
 from fern_python.generator_exec_wrapper import GeneratorExecWrapper
 

diff --git a/...rs/python/src/fern_python/codegen/ast/nodes/declarations/function/function_declaration.py b/...rs/python/src/fern_python/codegen/ast/nodes/declarations/function/function_declaration.py
@@ -20,8 +20,8 @@ def __init__(
         name: str,
         signature: FunctionSignature,
         body: CodeWriter,
-        overloads: Sequence[FunctionSignature] = None,
-        decorators: Sequence[AstNode] = None,
+        overloads: Optional[Sequence[FunctionSignature]] = None,
+        decorators: Optional[Sequence[AstNode]] = None,
         docstring: Optional[CodeWriter] = None,
         is_async: Optional[bool] = False,
     ):

diff --git a/...tors/python/src/fern_python/codegen/ast/nodes/declarations/function/function_signature.py b/...tors/python/src/fern_python/codegen/ast/nodes/declarations/function/function_signature.py
@@ -10,11 +10,11 @@ class FunctionSignature(AstNode):
     def __init__(
         self,
         *,
-        parameters: Sequence[FunctionParameter] = None,
+        parameters: Optional[Sequence[FunctionParameter]] = None,
         include_args: bool = False,
-        named_parameters: Sequence[NamedFunctionParameter] = None,
+        named_parameters: Optional[Sequence[NamedFunctionParameter]] = None,
         include_kwargs: bool = False,
-        return_type: TypeHint = None,
+        return_type: Optional[TypeHint] = None,
     ):
         self.parameters = parameters or []
         self.include_args = include_args

diff --git a/...python/codegen/ast/nodes/expressions/dictionary_instantiation/dictionary_instantiation.py b/...python/codegen/ast/nodes/expressions/dictionary_instantiation/dictionary_instantiation.py
@@ -8,7 +8,7 @@
 class DictionaryInstantiation(AstNode):
     def __init__(
         self,
-        entries: Sequence[Tuple[Expression, Expression]] = None,
+        entries: Optional[Sequence[Tuple[Expression, Expression]]] = None,
     ):
         self.entries = entries or []
 

diff --git a/generators/python/src/fern_python/codegen/ast/nodes/type_hint/type_hint.py b/generators/python/src/fern_python/codegen/ast/nodes/type_hint/type_hint.py
@@ -20,8 +20,8 @@ class TypeHint(AstNode):
     def __init__(
         self,
         type: Union[ClassReference, GenericTypeVar],
-        type_parameters: Sequence[TypeParameter] = None,
-        arguments: Sequence[Expression] = None,
+        type_parameters: Optional[Sequence[TypeParameter]] = None,
+        arguments: Optional[Sequence[Expression]] = None,
         is_optional: bool = False,
         is_literal: bool = False,
     ):

diff --git a/generators/python/src/fern_python/codegen/project.py b/generators/python/src/fern_python/codegen/project.py
@@ -7,7 +7,7 @@
 from types import TracebackType
 from typing import List, Optional, Sequence, Set, Type
 
-from fern.generator_exec.resources import GithubOutputMode, LicenseConfig, PypiMetadata
+from fern.generator_exec import GithubOutputMode, LicenseConfig, PypiMetadata
 from isort import file
 
 from fern_python.codegen import AST
@@ -39,7 +39,7 @@ def __init__(
         filepath: str,
         relative_path_to_project: str,
         python_version: str = "^3.8",
-        project_config: ProjectConfig = None,
+        project_config: Optional[ProjectConfig] = None,
         should_format_files: bool,
         sorted_modules: Optional[Sequence[str]] = None,
         flat_layout: bool = False,

diff --git a/generators/python/src/fern_python/codegen/pyproject_toml.py b/generators/python/src/fern_python/codegen/pyproject_toml.py
@@ -7,7 +7,7 @@
 from dataclasses import dataclass
 from typing import List, Optional, Set, cast
 
-from fern.generator_exec.resources import (
+from fern.generator_exec import (
     BasicLicense,
     GithubOutputMode,
     LicenseConfig,

diff --git a/generators/python/src/fern_python/codegen/reference_resolver_impl.py b/generators/python/src/fern_python/codegen/reference_resolver_impl.py
@@ -68,7 +68,8 @@ def resolve_references(self) -> None:
                     import_ = dataclasses.replace(
                         import_,
                         alias=construct_import_alias_for_collision(original_reference.import_),
-                    )
+                    )  # type: ignore
+                    # see https://github.com/python/mypy/pull/15962 for the mypy issue
 
                 self._original_import_to_resolved_import[original_reference.import_] = ResolvedImport(
                     import_=import_,

diff --git a/generators/python/src/fern_python/codegen/source_file.py b/generators/python/src/fern_python/codegen/source_file.py
@@ -59,7 +59,7 @@ def __init__(
         module_path: AST.ModulePath,
         reference_resolver: ReferenceResolverImpl,
         dependency_manager: DependencyManager,
-        completion_listener: Callable[[SourceFileImpl], None] = None,
+        completion_listener: Optional[Callable[[SourceFileImpl], None]] = None,
         should_format: bool,
         should_format_as_snippet: bool = False,
         should_include_header: bool = True,
@@ -136,7 +136,7 @@ class LocalClassReferenceImpl(LocalClassReference):
             def add_class_declaration(
                 class_reference_self,
                 declaration: AST.ClassDeclaration,
-                should_export: bool = None,
+                should_export: Optional[bool] = None,
             ) -> LocalClassReference:
                 return LocalClassReferenceImpl(
                     qualified_name_excluding_import=(

diff --git a/generators/python/src/fern_python/generator_cli/generator_cli.py b/generators/python/src/fern_python/generator_cli/generator_cli.py
@@ -4,7 +4,7 @@
 from math import e
 from typing import Dict, List, Optional, Union
 
-import fern.generator_exec.resources as generator_exec
+import fern.generator_exec as generator_exec
 import fern.ir.resources as ir_types
 import generatorcli
 import yaml  # type: ignore

diff --git a/generators/python/src/fern_python/generator_cli/readme_snippet_builder.py b/generators/python/src/fern_python/generator_cli/readme_snippet_builder.py
@@ -1,6 +1,6 @@
 from typing import Dict, List, Optional, Union
 
-import fern.generator_exec.resources as generator_exec
+import fern.generator_exec as generator_exec
 import fern.ir.resources as ir_types
 import generatorcli
 
@@ -289,7 +289,7 @@ def _get_user_specified_endpoint_ids_for_feature(
         return None
 
     def _get_feature_key(self, feature_id: generatorcli.feature.FeatureId) -> ir_types.FeatureId:
-        return ir_types.FeatureId.from_str(self._snake_to_camel_case(feature_id))
+        return self._snake_to_camel_case(feature_id)
 
     def _get_default_endpoint_id(
         self,
@@ -321,7 +321,7 @@ def _get_default_endpoint_id(
         if default_endpoint.id.identifier_override is None:
             raise RuntimeError("Internal error; all endpoints must define an endpoint id to generate README.md")
 
-        return ir_types.EndpointId.from_str(default_endpoint.id.identifier_override)
+        return default_endpoint.id.identifier_override
 
     def _build_endpoints(
         self, ir: ir_types.IntermediateRepresentation
@@ -332,7 +332,7 @@ def _build_endpoint_snippet_map(
         self, snippets: generator_exec.Snippets
     ) -> Dict[ir_types.EndpointId, generator_exec.PythonEndpointSnippet]:
         return {
-            ir_types.EndpointId.from_str(endpoint.id.identifier_override): self._get_python_endpoint_snippet(endpoint)
+            endpoint.id.identifier_override: self._get_python_endpoint_snippet(endpoint)
             for endpoint in snippets.endpoints
             if endpoint.id.identifier_override is not None
         }

diff --git a/generators/python/src/fern_python/generator_cli/reference_config_builder.py b/generators/python/src/fern_python/generator_cli/reference_config_builder.py
@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-import fern.generator_exec.resources as generator_exec
+import fern.generator_exec as generator_exec
 import fern.ir.resources as ir_types
 import generatorcli
 
@@ -34,7 +34,7 @@ def _convert_endpoint_metadata_to_title(
         self, endpoint_metadata: EndpointMetadata, has_parameters: bool
     ) -> generatorcli.reference.MethodInvocationSnippet:
         return generatorcli.reference.MethodInvocationSnippet(
-            snippetParts=[
+            snippet_parts=[
                 generatorcli.reference.LinkedText(text="client."),
                 generatorcli.reference.LinkedText(text=endpoint_metadata.endpoint_package_path),
                 generatorcli.reference.LinkedText(
@@ -149,9 +149,7 @@ def __init__(
         for endpoint_snippet in snippets.endpoints:
             endpoint_snippet_snippet = endpoint_snippet.snippet.get_as_union()
             if endpoint_snippet_snippet.type == "python" and endpoint_snippet.id.identifier_override is not None:
-                self._endpoint_snippets[
-                    ir_types.EndpointId.from_str(endpoint_snippet.id.identifier_override)
-                ] = endpoint_snippet_snippet.sync_client
+                self._endpoint_snippets[endpoint_snippet.id.identifier_override] = endpoint_snippet_snippet.sync_client
 
     def get_root_package_location(self) -> str:
         return self._project.get_relative_source_file_filepath(
@@ -208,7 +206,7 @@ def generate_reference_config(self) -> generatorcli.reference.ReferenceConfig:
             )
 
         return generatorcli.reference.ReferenceConfig(
-            rootSection=root_reference_section,
+            root_section=root_reference_section,
             sections=self.reference_config_sections,
             language="PYTHON",
         )
diff --git a/generators/python/src/fern_python/generator_exec_wrapper/generator_exec_wrapper.py b/generators/python/src/fern_python/generator_exec_wrapper/generator_exec_wrapper.py
@@ -3,12 +3,8 @@
 from fdr import FdrClientEnvironment
 from fdr.client import FdrClient
 from fern.generator_exec.client import FernGeneratorExec
-from fern.generator_exec.resources.config import (
-    GeneratorConfig,
-    RemoteGeneratorEnvironment,
-)
-from fern.generator_exec.resources.logging import GeneratorUpdate, TaskId
-from fern.generator_exec.resources.readme import GenerateReadmeRequest
+from fern.generator_exec.config import GeneratorConfig, RemoteGeneratorEnvironment
+from fern.generator_exec.logging import GeneratorUpdate, TaskId
 
 
 class GeneratorExecWrapper:
@@ -19,7 +15,7 @@ def __init__(self, generator_config: GeneratorConfig):
         generator_config.environment.visit(local=lambda: (), remote=lambda env: self._init_remote(env))
 
     def _init_remote(self, env: RemoteGeneratorEnvironment) -> None:
-        self.generator_exec_client = FernGeneratorExec(environment=env.coordinator_url_v_2)
+        self.generator_exec_client = FernGeneratorExec(base_url=env.coordinator_url_v_2)
         self.task_id = env.id
         if self._is_in_development(env):
             self.fdr_client = FdrClient(base_url="https://registry-dev2.buildwithfern.com")
@@ -35,10 +31,3 @@ def send_update(self, generator_update: GeneratorUpdate) -> None:
     def send_updates(self, generator_updates: typing.List[GeneratorUpdate]) -> None:
         if self.generator_exec_client is not None and self.task_id is not None:
             self.generator_exec_client.logging.send_update(task_id=self.task_id, request=generator_updates)
-
-    # Returns if the request was actually sent
-    def generate_readme(self, generate_readme_request: GenerateReadmeRequest) -> bool:
-        if self.generator_exec_client is not None and self.task_id is not None:
-            self.generator_exec_client.readme.generate_readme(task_id=self.task_id, request=generate_readme_request)
-            return True
-        return False
diff --git a/generators/python/src/fern_python/generators/context/pydantic_generator_context.py b/generators/python/src/fern_python/generators/context/pydantic_generator_context.py
@@ -4,7 +4,7 @@
 from typing import Callable, List, Optional, Set
 
 import fern.ir.resources as ir_types
-from fern.generator_exec.resources import GeneratorConfig
+from fern.generator_exec import GeneratorConfig
 
 from fern_python.codegen import AST, Filepath
 from fern_python.declaration_referencer import AbstractDeclarationReferencer

diff --git a/generators/python/src/fern_python/generators/context/pydantic_generator_context_impl.py b/generators/python/src/fern_python/generators/context/pydantic_generator_context_impl.py
@@ -1,7 +1,7 @@
 from typing import Callable, List, Optional, Set
 
 import fern.ir.resources as ir_types
-from fern.generator_exec.resources import GeneratorConfig
+from fern.generator_exec import GeneratorConfig
 
 from fern_python.codegen import AST, Filepath
 from fern_python.declaration_referencer import AbstractDeclarationReferencer