fix: remove python version upper bound

fixes #944

the main fix here is switching from `cattr.register_structure_hook` to
`cattr.register_structure_hook_func`. I re-organized some of the code
in an effort to reproduce the issue in the test_serialize.py suite but
I could not reproduce there, only in integration tests.

Now that it's a function, we don't need a generated hook per type.

I further isolated the converter usage switching the serializing to use
each 31/40 dedicated converter.

I also switched to the newer GenConverter.

setup testing against python 3.10

.github/workflows/python-ci.yml

@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.9.9
+          python-version: '3.10'
       - run: pip install -e .
       - run: pip install mypy types-requests
       - run: mypy looker_sdk/
@@ -59,14 +59,16 @@ jobs:
           - macos
           - windows
         python-version:
-          - '3.9.9'
+          - '3.10'
         include:
           - python-version: '3.6'
             os: ubuntu
           - python-version: '3.7'
             os: ubuntu
           - python-version: '3.8'
             os: ubuntu
+          - python-version: '3.9'
+            os: ubuntu
 
     steps:
       - name: Repo Checkout
@@ -185,7 +187,7 @@ jobs:
           pip install tox
 
       - name: Determine credentials version
-        # Prior to 21_18, each version had different credentials and a 
+        # Prior to 21_18, each version had different credentials and a
         # different secret. 21_20 and later all use the same credentials
         # as 21_18. The parse_version.sh script parses the version and
         # yields 21_12, 21_14, 21_16 etc for those versions but 21_18 for
@@ -229,7 +231,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.9.9
+          python-version: '3.10'
       - name: Twine upload check
         run: |
           pip install wheel twine

.github/workflows/python-publish.yml

@@ -29,7 +29,7 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: 3.9.9
+          python-version: 3.10
 
       - name: Package release artifacts
         run: |

packages/sdk-codegen/src/python.gen.spec.ts

@@ -88,19 +88,15 @@ class LookerSDK(api_methods.APIMethods):
       const type = apiTestModel.types.LookmlModelExploreJoins
       gen.declareType(indent, type)
       expect(gen.modelsEpilogue('')).toEqual(`
-
-# The following cattrs structure hook registrations are a workaround
-# for https://github.com/Tinche/cattrs/pull/42 Once this issue is resolved
-# these calls will be removed.
-
 import functools  # noqa:E402
 
-forward_ref_structure_hook = functools.partial(sr.forward_ref_structure_hook, globals(), sr.converter)
-translate_keys_structure_hook = functools.partial(sr.translate_keys_structure_hook, sr.converter)
-sr.converter.register_structure_hook(
-    ForwardRef("LookmlModelExploreJoins"),  # type: ignore
-    forward_ref_structure_hook  # type:ignore
+forward_ref_structure_hook = functools.partial(
+    sr.forward_ref_structure_hook, globals(), sr.converter
+)
+sr.converter.register_structure_hook_func(
+    lambda t: t.__class__ is ForwardRef, forward_ref_structure_hook
 )
+translate_keys_structure_hook = functools.partial(sr.translate_keys_structure_hook, sr.converter)
 sr.converter.register_structure_hook(
     LookmlModelExploreJoins,  # type: ignore
     translate_keys_structure_hook  # type:ignore

packages/sdk-codegen/src/python.gen.ts

@@ -100,7 +100,6 @@ export class PythonGen extends CodeGen {
 
   // cattrs [un]structure hooks for model [de]serialization
   hooks: string[] = []
-  structureHookFR = 'forward_ref_structure_hook'
   structureHookTK = 'translate_keys_structure_hook'
   pythonReservedKeywordClasses: Set<string> = new Set()
 
@@ -141,18 +140,14 @@ DelimSequence = model.DelimSequence
 `
 
   modelsEpilogue = (_indent: string) => `
-
-# The following cattrs structure hook registrations are a workaround
-# for https://github.com/Tinche/cattrs/pull/42 Once this issue is resolved
-# these calls will be removed.
-
 import functools  # noqa:E402
 
-${
-  this.structureHookFR
-} = functools.partial(sr.forward_ref_structure_hook, globals(), sr.converter${
-    this.apiRef
-  })
+forward_ref_structure_hook = functools.partial(
+    sr.forward_ref_structure_hook, globals(), sr.converter${this.apiRef}
+)
+sr.converter${this.apiRef}.register_structure_hook_func(
+    lambda t: t.__class__ is ForwardRef, forward_ref_structure_hook
+)
 ${
   this.structureHookTK
 } = functools.partial(sr.translate_keys_structure_hook, sr.converter${
@@ -476,10 +471,6 @@ ${this.hooks.join('\n')}
       }
     }
 
-    const forwardRef = `ForwardRef("${type.name}")`
-    this.hooks.push(
-      `sr.converter${this.apiRef}.register_structure_hook(\n${bump}${forwardRef},  # type: ignore\n${bump}${this.structureHookFR}  # type:ignore\n)`
-    )
     if (usesReservedPythonKeyword) {
       this.hooks.push(
         `sr.converter${this.apiRef}.register_structure_hook(\n${bump}${type.name},  # type: ignore\n${bump}${this.structureHookTK}  # type:ignore\n)`

python/.python-version

@@ -1,3 +1,5 @@
+3.10.2
+3.9.10
 3.9.9
 3.9.0
 3.8.2

python/Pipfile

@@ -31,8 +31,7 @@ cattrs = ">=1.3"
 python-dateutil = "*"
 
 [requires]
-# pin python at 3.9.9 due to https://github.com/looker-open-source/sdk-codegen/issues/944
-python_full_version = "3.9.9"
+python_full_version = "3.10"
 
 [pipenv]
 # for `black`

python/looker_sdk/__init__.py

@@ -62,7 +62,7 @@ def init31(
     return methods31.Looker31SDK(
         auth_session.AuthSession(settings, transport, serialize.deserialize31, "3.1"),
         serialize.deserialize31,
-        serialize.serialize,
+        serialize.serialize31,
         transport,
         "3.1",
     )
@@ -82,7 +82,7 @@ def init40(
     return methods40.Looker40SDK(
         auth_session.AuthSession(settings, transport, serialize.deserialize40, "4.0"),
         serialize.deserialize40,
-        serialize.serialize,
+        serialize.serialize40,
         transport,
         "4.0",
     )

python/looker_sdk/rtl/api_methods.py

@@ -147,7 +147,7 @@ def _get_serialized(self, body: TBody) -> Optional[bytes]:
         if isinstance(body, str):
             serialized = body.encode("utf-8")
         elif isinstance(body, (list, dict, model.Model)):
-            serialized = self.serialize(body)
+            serialized = self.serialize(api_model=body)  # type: ignore
         else:
             serialized = None
         return serialized

python/looker_sdk/rtl/auth_session.py

@@ -317,7 +317,7 @@ def _request_token(
         response = self.transport.request(
             transport.HttpMethod.POST,
             urllib.parse.urljoin(self.settings.base_url, "/api/token"),
-            body=self.serialize(grant_type),
+            body=self.serialize(api_model=grant_type),  # type: ignore
         )
         if not response.ok:
             raise error.SDKError(response.value.decode(encoding=response.encoding))

python/looker_sdk/rtl/hooks.py

@@ -0,0 +1,86 @@
+# The MIT License (MIT)
+#
+# Copyright (c) 2022 Looker Data Sciences, Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import datetime
+import enum
+import keyword
+import sys
+from typing import Type
+
+
+def unstructure_hook(converter, api_model):
+    """cattr unstructure hook
+
+    Map reserved_ words in models to correct json field names.
+    Also handle stripping None fields from dict while setting
+    EXPLICIT_NULL fields to None so that we only send null
+    in the json for fields the caller set EXPLICIT_NULL on.
+    """
+    data = converter.unstructure_attrs_asdict(api_model)
+    for key, value in data.copy().items():
+        if value is None:
+            del data[key]
+        elif value == "EXPLICIT_NULL":
+            data[key] = None
+        # bug here: in the unittests cattrs unstructures this correctly
+        # as an enum calling .value but in the integration tests we see
+        # it doesn't for WriteCreateQueryTask.result_format for some reason
+        # Haven't been able to debug it fully, so catching and processing
+        # it here.
+        elif isinstance(value, enum.Enum):
+            data[key] = value.value
+    for reserved in keyword.kwlist:
+        if f"{reserved}_" in data:
+            data[reserved] = data.pop(f"{reserved}_")
+    return data
+
+
+DATETIME_FMT = "%Y-%m-%dT%H:%M:%S.%f%z"
+if sys.version_info < (3, 7):
+    from dateutil import parser
+
+    def datetime_structure_hook(
+        d: str, t: Type[datetime.datetime]
+    ) -> datetime.datetime:
+        return parser.isoparse(d)
+
+else:
+
+    def datetime_structure_hook(
+        d: str, t: Type[datetime.datetime]
+    ) -> datetime.datetime:
+        return datetime.datetime.strptime(d, DATETIME_FMT)
+
+
+def datetime_unstructure_hook(dt):
+    return dt.strftime(DATETIME_FMT)
+
+
+def tr_data_keys(data):
+    """Map top level json keys to model property names.
+
+    Currently this translates reserved python keywords like "from" => "from_"
+    """
+    for reserved in keyword.kwlist:
+        if reserved in data and isinstance(data, dict):
+            data[f"{reserved}_"] = data.pop(reserved)
+    return data

python/looker_sdk/rtl/model.py

@@ -24,28 +24,41 @@
 """
 
 import collections
+import datetime
 import enum
+import functools
 import keyword
-from typing import Any, cast, Iterable, Sequence, Optional, TypeVar
+from typing import Any, Iterable, Optional, Sequence, TypeVar, cast
+
+import cattr
+
+from looker_sdk.rtl import hooks
 
 try:
     from typing import ForwardRef  # type: ignore
 except ImportError:
     from typing import _ForwardRef as ForwardRef  # type: ignore
 
-import cattr
-
 
 EXPLICIT_NULL = cast(Any, "EXPLICIT_NULL")  # type:ignore
 
 
 class Model:
-    """Base model for all generated models.
-    """
+    """Base model for all generated models."""
+
+    def _get_converter(self):
+        if not hasattr(self, "_converter"):
+            converter = cattr.Converter()
+            converter.register_unstructure_hook(
+                datetime.datetime, hooks.datetime_unstructure_hook
+            )
+            uh = functools.partial(hooks.unstructure_hook, converter)
+            converter.register_unstructure_hook(Model, uh)  # type: ignore
+            self._converter = converter
+        return self._converter
 
     def _key_to_attr(self, key):
-        """Appends the trailing _ to python reserved words.
-        """
+        """Appends the trailing _ to python reserved words."""
         if key[-1] == "_":
             raise KeyError(key)
         if key in keyword.kwlist:
@@ -95,7 +108,7 @@ def err(val):
                     raise ValueError(err(value))
                 value = enum_member
             elif issubclass(actual_type, Model):
-                value = cattr.structure(value, actual_type)
+                value = self._get_converter().structure(value, actual_type)
 
         return setattr(self, key, value)
 
@@ -104,22 +117,22 @@ def __delitem__(self, key):
         setattr(self, self._key_to_attr(key), None)
 
     def __iter__(self):
-        return iter(cattr.unstructure(self))
+        return iter(self._get_converter().unstructure(self))
 
     def __len__(self):
-        return len(cattr.unstructure(self))
+        return len(self._get_converter().unstructure(self))
 
     def __contains__(self, key):
-        return key in cattr.unstructure(self)
+        return key in self._get_converter().unstructure(self)
 
     def keys(self):
-        return cattr.unstructure(self).keys()
+        return self._get_converter().unstructure(self).keys()
 
     def items(self):
-        return cattr.unstructure(self).items()
+        return self._get_converter().unstructure(self).items()
 
     def values(self):
-        return cattr.unstructure(self).values()
+        return self._get_converter().unstructure(self).values()
 
     def get(self, key, default=None):
         try: