From 9522581eca5003c073e5e2aaaa0ce25fde149ab0 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Thu, 3 Aug 2023 13:01:41 -0500 Subject: [PATCH 01/12] feat: Add additional parameters to `to_json()` and `to_dict()` methods - These parameters are just passthroughs to [`MessageToJson`][1] and [`MessageToDict`][2] [1]: https://googleapis.dev/python/protobuf/latest/google/protobuf/json_format.html#google.protobuf.json_format.MessageToJson [2]: https://googleapis.dev/python/protobuf/latest/google/protobuf/json_format.html#google.protobuf.json_format.MessageToDict --- proto/message.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/proto/message.py b/proto/message.py index 253e6240..4133b062 100644 --- a/proto/message.py +++ b/proto/message.py @@ -376,7 +376,10 @@ def to_json( use_integers_for_enums=True, including_default_value_fields=True, preserving_proto_field_name=False, + sort_keys=False, indent=2, + float_precision=None, + ensure_ascii=False, ) -> str: """Given a message instance, serialize it to json @@ -389,10 +392,16 @@ def to_json( preserving_proto_field_name (Optional(bool)): An option that determines whether field name representations preserve proto case (snake_case) or use lowerCamelCase. Default is False. - indent: The JSON object will be pretty-printed with this indent level. + sort_keys (Optional(bool)): If True, then the output will be sorted by field names. + Default is False. + indent (int): The JSON object will be pretty-printed with this indent level. An indent level of 0 or negative will only insert newlines. Pass None for the most compact representation without newlines. - + float_precision (Optional(int)): If set, use this to specify float field valid digits. + Default is None. + ensure_ascii (Optional(bool)): If True, strings with non-ASCII characters are escaped. + If False, Unicode strings are returned unchanged. + Default is False. Returns: str: The json string representation of the protocol buffer. """ @@ -401,7 +410,10 @@ def to_json( use_integers_for_enums=use_integers_for_enums, including_default_value_fields=including_default_value_fields, preserving_proto_field_name=preserving_proto_field_name, + sort_keys=sort_keys, indent=indent, + float_precision=float_precision, + ensure_ascii=ensure_ascii, ) def from_json(cls, payload, *, ignore_unknown_fields=False) -> "Message": @@ -428,6 +440,7 @@ def to_dict( use_integers_for_enums=True, preserving_proto_field_name=True, including_default_value_fields=True, + float_precision=None, ) -> "Message": """Given a message instance, return its representation as a python dict. @@ -443,6 +456,8 @@ def to_dict( including_default_value_fields (Optional(bool)): An option that determines whether the default field values should be included in the results. Default is True. + float_precision (Optional(int)): If set, use this to specify float field valid digits. + Default is None. Returns: dict: A representation of the protocol buffer using pythonic data structures. @@ -454,6 +469,7 @@ def to_dict( including_default_value_fields=including_default_value_fields, preserving_proto_field_name=preserving_proto_field_name, use_integers_for_enums=use_integers_for_enums, + float_precision=float_precision, ) def copy_from(cls, instance, other): From a81ee7c95d619c107418be65d59d0f79fd5feebe Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 4 Aug 2023 11:54:51 -0500 Subject: [PATCH 02/12] chore: Update min protobuf version to 3.20.0 to support `ensure_ascii` --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 670bca40..b9e1acf2 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ long_description=README, platforms="Posix; MacOS X", include_package_data=True, - install_requires=("protobuf >= 3.19.0, <5.0.0dev",), + install_requires=("protobuf >= 3.20.0, <5.0.0dev",), extras_require={ "testing": [ "google-api-core[grpc] >= 1.31.5", From 4b544642c340cb080339d5f236377b0918fbc927 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 4 Aug 2023 11:59:18 -0500 Subject: [PATCH 03/12] Removed `ensure_ascii` to prevent version compatibility errors --- proto/message.py | 5 ----- setup.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/proto/message.py b/proto/message.py index 4133b062..413e8d1c 100644 --- a/proto/message.py +++ b/proto/message.py @@ -379,7 +379,6 @@ def to_json( sort_keys=False, indent=2, float_precision=None, - ensure_ascii=False, ) -> str: """Given a message instance, serialize it to json @@ -399,9 +398,6 @@ def to_json( Pass None for the most compact representation without newlines. float_precision (Optional(int)): If set, use this to specify float field valid digits. Default is None. - ensure_ascii (Optional(bool)): If True, strings with non-ASCII characters are escaped. - If False, Unicode strings are returned unchanged. - Default is False. Returns: str: The json string representation of the protocol buffer. """ @@ -413,7 +409,6 @@ def to_json( sort_keys=sort_keys, indent=indent, float_precision=float_precision, - ensure_ascii=ensure_ascii, ) def from_json(cls, payload, *, ignore_unknown_fields=False) -> "Message": diff --git a/setup.py b/setup.py index b9e1acf2..670bca40 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ long_description=README, platforms="Posix; MacOS X", include_package_data=True, - install_requires=("protobuf >= 3.20.0, <5.0.0dev",), + install_requires=("protobuf >= 3.19.0, <5.0.0dev",), extras_require={ "testing": [ "google-api-core[grpc] >= 1.31.5", From c9a990c376ec93ae794b4d2227a0fe8194c7c587 Mon Sep 17 00:00:00 2001 From: Holt Skinner <13262395+holtskinner@users.noreply.github.com> Date: Fri, 1 Sep 2023 10:04:29 -0700 Subject: [PATCH 04/12] Update proto/message.py Co-authored-by: Anthonios Partheniou --- proto/message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto/message.py b/proto/message.py index 413e8d1c..7232d42f 100644 --- a/proto/message.py +++ b/proto/message.py @@ -393,7 +393,7 @@ def to_json( proto case (snake_case) or use lowerCamelCase. Default is False. sort_keys (Optional(bool)): If True, then the output will be sorted by field names. Default is False. - indent (int): The JSON object will be pretty-printed with this indent level. + indent (Optional(int)): The JSON object will be pretty-printed with this indent level. An indent level of 0 or negative will only insert newlines. Pass None for the most compact representation without newlines. float_precision (Optional(int)): If set, use this to specify float field valid digits. From c843a7748681fd9dd5b94b919d61aab06956813e Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 10:22:27 -0500 Subject: [PATCH 05/12] test: Add tests for `sort_keys` and `float_precision` --- tests/test_json.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_json.py b/tests/test_json.py index 93ca936c..8c22c90e 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest +import re import proto from google.protobuf.json_format import MessageToJson, Parse, ParseError @@ -172,3 +173,25 @@ class Squid(proto.Message): s_two = Squid.from_json(j) assert s == s_two + + +def test_json_sort_keys(): + class Squid(proto.Message): + name = proto.Field(proto.STRING, number=1) + mass_kg = proto.Field(proto.INT32, number=2) + + s = Squid(name="Steve", mass_kg=20) + j = Squid.to_json(s, sort_keys=True) + + assert re.match(r"massKg.|\s*name", j) + + +def test_json_float_precision(): + class Squid(proto.Message): + name = proto.Field(proto.STRING, number=1) + mass_kg = proto.Field(proto.FLOAT, number=2) + + s = Squid(name="Steve", mass_kg=3.14159265) + j = Squid.to_json(s, float_precision=2) + + assert j == '{"name":"Steve","massKg":3.14}' From ec014f61b0978f8cab27f8748db9bdcef1a3490a Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 10:30:30 -0500 Subject: [PATCH 06/12] Fix test checks --- tests/test_json.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_json.py b/tests/test_json.py index 8c22c90e..e66d1dae 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -183,7 +183,7 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=20) j = Squid.to_json(s, sort_keys=True) - assert re.match(r"massKg.|\s*name", j) + assert re.match(r"massKg.|\n*name", j) def test_json_float_precision(): @@ -192,6 +192,14 @@ class Squid(proto.Message): mass_kg = proto.Field(proto.FLOAT, number=2) s = Squid(name="Steve", mass_kg=3.14159265) - j = Squid.to_json(s, float_precision=2) + j = Squid.to_json(s, float_precision=3) - assert j == '{"name":"Steve","massKg":3.14}' + assert ( + j + == """ + { + "name": "Steve", + "massKg": 3.14 + } + """ + ) From f89017764c9733d26ae80f6478f9ebd9ddac5581 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 10:38:23 -0500 Subject: [PATCH 07/12] Remove indent from float_precision test and add re.compile to sortkeys regex --- tests/test_json.py | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/test_json.py b/tests/test_json.py index e66d1dae..07b5d99e 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -183,7 +183,7 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=20) j = Squid.to_json(s, sort_keys=True) - assert re.match(r"massKg.|\n*name", j) + assert re.match(re.compile(r"massKg.|\s*name"), j) def test_json_float_precision(): @@ -192,14 +192,6 @@ class Squid(proto.Message): mass_kg = proto.Field(proto.FLOAT, number=2) s = Squid(name="Steve", mass_kg=3.14159265) - j = Squid.to_json(s, float_precision=3) + j = Squid.to_json(s, float_precision=3, indent=None) - assert ( - j - == """ - { - "name": "Steve", - "massKg": 3.14 - } - """ - ) + assert j == '{"name":"Steve","massKg":3.14}' From 0137ab3eceb7ee2318c59e343f28bb31bc867390 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 10:45:10 -0500 Subject: [PATCH 08/12] Add spaces to float precision check, add dotall to sortkeys check --- tests/test_json.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_json.py b/tests/test_json.py index 07b5d99e..eeacdf9e 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -183,7 +183,7 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=20) j = Squid.to_json(s, sort_keys=True) - assert re.match(re.compile(r"massKg.|\s*name"), j) + assert re.match(r"massKg.*name", j, re.DOTALL) def test_json_float_precision(): @@ -194,4 +194,4 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=3.14159265) j = Squid.to_json(s, float_precision=3, indent=None) - assert j == '{"name":"Steve","massKg":3.14}' + assert j == '{"name": "Steve","massKg": 3.14}' From eaf901b19a11a7889247ed2eb9ee5dab5cc4e79d Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 13:14:57 -0500 Subject: [PATCH 09/12] Remove indent from sort_keys test, add space to float precision test --- tests/test_json.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_json.py b/tests/test_json.py index eeacdf9e..885ecf45 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -181,9 +181,9 @@ class Squid(proto.Message): mass_kg = proto.Field(proto.INT32, number=2) s = Squid(name="Steve", mass_kg=20) - j = Squid.to_json(s, sort_keys=True) + j = Squid.to_json(s, sort_keys=True, indent=None) - assert re.match(r"massKg.*name", j, re.DOTALL) + assert re.match(r"massKg.*name", j) def test_json_float_precision(): @@ -194,4 +194,4 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=3.14159265) j = Squid.to_json(s, float_precision=3, indent=None) - assert j == '{"name": "Steve","massKg": 3.14}' + assert j == '{"name": "Steve", "massKg": 3.14}' From 4b7b1e2a88a5fb8903c7a1b34a3935e379efc878 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Tue, 5 Sep 2023 13:37:29 -0500 Subject: [PATCH 10/12] Changed match to search --- tests/test_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_json.py b/tests/test_json.py index 885ecf45..4a579793 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -183,7 +183,7 @@ class Squid(proto.Message): s = Squid(name="Steve", mass_kg=20) j = Squid.to_json(s, sort_keys=True, indent=None) - assert re.match(r"massKg.*name", j) + assert re.search(r"massKg.*name", j) def test_json_float_precision(): From eea1fe6a811dcd561ee021fa0d75eb4ab18683f5 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Wed, 6 Sep 2023 10:24:42 -0500 Subject: [PATCH 11/12] test: Add test for `to_dict()` with `float_precision` --- tests/test_message.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_message.py b/tests/test_message.py index 3146f0bb..ce6905ee 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -271,6 +271,16 @@ class Color(proto.Enum): assert new_s == s +def test_serialize_to_dict_float_precision(): + class Squid(proto.Message): + mass_kg = proto.Field(proto.FLOAT, number=1) + + s = Squid(mass_kg=3.14159265) + + s_dict = Squid.to_dict(s, float_precision=3) + assert s_dict["mass_kg"] == 3.14 + + def test_unknown_field_deserialize(): # This is a somewhat common setup: a client uses an older proto definition, # while the server sends the newer definition. The client still needs to be From 54563264444a0694f17e51a8fa3240a7d65347b9 Mon Sep 17 00:00:00 2001 From: Holt Skinner Date: Fri, 8 Sep 2023 11:01:39 -0500 Subject: [PATCH 12/12] Added TODOs for `float_precision` issue --- tests/test_json.py | 1 + tests/test_message.py | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/test_json.py b/tests/test_json.py index 4a579793..e94e935a 100644 --- a/tests/test_json.py +++ b/tests/test_json.py @@ -186,6 +186,7 @@ class Squid(proto.Message): assert re.search(r"massKg.*name", j) +# TODO: https://github.com/googleapis/proto-plus-python/issues/390 def test_json_float_precision(): class Squid(proto.Message): name = proto.Field(proto.STRING, number=1) diff --git a/tests/test_message.py b/tests/test_message.py index ce6905ee..983cde82 100644 --- a/tests/test_message.py +++ b/tests/test_message.py @@ -271,6 +271,7 @@ class Color(proto.Enum): assert new_s == s +# TODO: https://github.com/googleapis/proto-plus-python/issues/390 def test_serialize_to_dict_float_precision(): class Squid(proto.Message): mass_kg = proto.Field(proto.FLOAT, number=1)