Skip to content

Commit

Permalink
Revert "feat: json serialization and deserialization support stringy …
Browse files Browse the repository at this point in the history 
…enums (#112)" (#116)

This reverts commit 8d2e3a3.
  • Loading branch information
@busunkim96
busunkim96 authored Sep 3, 2020
1 parent 23d940e commit 91c6d7b
Show file tree
Hide file tree
Showing 13 changed files with 78 additions and 373 deletions.
15 changes: 5 additions & 10 deletions noxfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,11 @@ def unit(session, proto="python"):
"py.test",
"-W=error",
"--quiet",
*(
session.posargs # Coverage info when running individual tests is annoying.
or [
"--cov=proto",
"--cov-config=.coveragerc",
"--cov-report=term",
"--cov-report=html",
os.path.join("tests", ""),
]
),
"--cov=proto",
"--cov-config=.coveragerc",
"--cov-report=term",
"--cov-report=html",
os.path.join("tests", ""),
)


Expand Down
44 changes: 5 additions & 39 deletions proto/_file_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import collections.abc
import inspect
import logging

from google.protobuf import descriptor_pb2
from google.protobuf import descriptor_pool
from google.protobuf import message
from google.protobuf import reflection
Expand All @@ -28,33 +27,11 @@

class _FileInfo(
collections.namedtuple(
"_FileInfo",
["descriptor", "messages", "enums", "name", "nested", "nested_enum"],
"_FileInfo", ["descriptor", "messages", "enums", "name", "nested"]
)
):
registry = {} # Mapping[str, '_FileInfo']

@classmethod
def maybe_add_descriptor(cls, filename, package):
descriptor = cls.registry.get(filename)
if not descriptor:
descriptor = cls.registry[filename] = cls(
descriptor=descriptor_pb2.FileDescriptorProto(
name=filename, package=package, syntax="proto3",
),
enums=collections.OrderedDict(),
messages=collections.OrderedDict(),
name=filename,
nested={},
nested_enum={},
)

return descriptor

@staticmethod
def proto_file_name(name):
return "{0}.proto".format(name.replace(".", "/"))

def _get_manifest(self, new_class):
module = inspect.getmodule(new_class)
if hasattr(module, "__protobuf__"):
Expand Down Expand Up @@ -130,13 +107,6 @@ def generate_file_pb(self, new_class, fallback_salt=""):
for field in proto_plus_message._meta.fields.values():
if field.message and isinstance(field.message, str):
field.message = self.messages[field.message]
elif field.enum and isinstance(field.enum, str):
field.enum = self.enums[field.enum]

# Same thing for enums
for full_name, proto_plus_enum in self.enums.items():
descriptor = pool.FindEnumTypeByName(full_name)
proto_plus_enum._meta.pb = descriptor

# We no longer need to track this file's info; remove it from
# the module's registry and from this object.
Expand All @@ -160,16 +130,14 @@ def ready(self, new_class):
"""
# If there are any nested descriptors that have not been assigned to
# the descriptors that should contain them, then we are not ready.
if len(self.nested) or len(self.nested_enum):
if len(self.nested):
return False

# If there are any unresolved fields (fields with a composite message
# declared as a string), ensure that the corresponding message is
# declared.
for field in self.unresolved_fields:
if (field.message and field.message not in self.messages) or (
field.enum and field.enum not in self.enums
):
if field.message not in self.messages:
return False

# If the module in which this class is defined provides a
Expand All @@ -188,7 +156,5 @@ def unresolved_fields(self):
"""Return fields with referencing message types as strings."""
for proto_plus_message in self.messages.values():
for field in proto_plus_message._meta.fields.values():
if (field.message and isinstance(field.message, str)) or (
field.enum and isinstance(field.enum, str)
):
if field.message and isinstance(field.message, str):
yield field
55 changes: 0 additions & 55 deletions proto/enums.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,6 @@

import enum

from google.protobuf import descriptor_pb2

from proto import _file_info
from proto import _package_info
from proto.marshal.rules.enums import EnumRule

Expand All @@ -33,58 +30,12 @@ def __new__(mcls, name, bases, attrs):
# this component belongs within the file.
package, marshal = _package_info.compile(name, attrs)

# Determine the local path of this proto component within the file.
local_path = tuple(attrs.get("__qualname__", name).split("."))

# Sanity check: We get the wrong full name if a class is declared
# inside a function local scope; correct this.
if "<locals>" in local_path:
ix = local_path.index("<locals>")
local_path = local_path[: ix - 1] + local_path[ix + 1 :]

# Determine the full name in protocol buffers.
full_name = ".".join((package,) + local_path).lstrip(".")
filename = _file_info._FileInfo.proto_file_name(
attrs.get("__module__", name.lower())
)
enum_desc = descriptor_pb2.EnumDescriptorProto(
name=name,
# Note: the superclass ctor removes the variants, so get them now.
# Note: proto3 requires that the first variant value be zero.
value=sorted(
(
descriptor_pb2.EnumValueDescriptorProto(name=name, number=number)
# Minor hack to get all the enum variants out.
for name, number in attrs.items()
if isinstance(number, int)
),
key=lambda v: v.number,
),
)

file_info = _file_info._FileInfo.maybe_add_descriptor(filename, package)
if len(local_path) == 1:
file_info.descriptor.enum_type.add().MergeFrom(enum_desc)
else:
file_info.nested_enum[local_path] = enum_desc

# Run the superclass constructor.
cls = super().__new__(mcls, name, bases, attrs)

# We can't just add a "_meta" element to attrs because the Enum
# machinery doesn't know what to do with a non-int value.
# The pb is set later, in generate_file_pb
cls._meta = _EnumInfo(full_name=full_name, pb=None)

file_info.enums[full_name] = cls

# Register the enum with the marshal.
marshal.register(cls, EnumRule(cls))

# Generate the descriptor for the file if it is ready.
if file_info.ready(new_class=cls):
file_info.generate_file_pb(new_class=cls, fallback_salt=full_name)

# Done; return the class.
return cls

Expand All @@ -93,9 +44,3 @@ class Enum(enum.IntEnum, metaclass=ProtoEnumMeta):
"""A enum object that also builds a protobuf enum descriptor."""

pass


class _EnumInfo:
def __init__(self, *, full_name: str, pb):
self.full_name = full_name
self.pb = pb
25 changes: 16 additions & 9 deletions proto/fields.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ def __init__(
def descriptor(self):
"""Return the descriptor for the field."""
if not self._descriptor:
proto_type = self.proto_type
# Resolve the message type, if any, to a string.
type_name = None
if isinstance(self.message, str):
Expand All @@ -84,23 +85,29 @@ def descriptor(self):
type_name = (
self.message.DESCRIPTOR.full_name
if hasattr(self.message, "DESCRIPTOR")
else self.message._meta.full_name
else self.message.meta.full_name
)
elif isinstance(self.enum, str):
if not self.enum.startswith(self.package):
self.enum = "{package}.{name}".format(
package=self.package, name=self.enum,
)
type_name = self.enum
elif self.enum:
type_name = self.enum._meta.full_name
# Nos decipiat.
#
# As far as the wire format is concerned, enums are int32s.
# Protocol buffers itself also only sends ints; the enum
# objects are simply helper classes for translating names
# and values and it is the user's job to resolve to an int.
#
# Therefore, the non-trivial effort of adding the actual
# enum descriptors seems to add little or no actual value.
#
# FIXME: Eventually, come back and put in the actual enum
# descriptors.
proto_type = ProtoType.INT32

# Set the descriptor.
self._descriptor = descriptor_pb2.FieldDescriptorProto(
name=self.name,
number=self.number,
label=3 if self.repeated else 1,
type=self.proto_type,
type=proto_type,
type_name=type_name,
json_name=self.json_name,
proto3_optional=self.optional,
Expand Down
42 changes: 18 additions & 24 deletions proto/message.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ def __new__(mcls, name, bases, attrs):
field_msg = field.message
if hasattr(field_msg, "pb") and callable(field_msg.pb):
field_msg = field_msg.pb()

# Sanity check: The field's message may not yet be defined if
# it was a Message defined in the same file, and the file
# descriptor proto has not yet been generated.
Expand All @@ -153,13 +154,7 @@ def __new__(mcls, name, bases, attrs):
# correctly when the file descriptor is created later.
if field_msg:
proto_imports.add(field_msg.DESCRIPTOR.file.name)

# Same thing, but for enums.
elif field.enum and not isinstance(field.enum, str):
field_enum = field.enum._meta.pb

if field_enum:
proto_imports.add(field_enum.file.name)
symbol_database.Default().RegisterMessage(field_msg)

# Increment the field index counter.
index += 1
Expand Down Expand Up @@ -188,13 +183,24 @@ def __new__(mcls, name, bases, attrs):
# Determine the filename.
# We determine an appropriate proto filename based on the
# Python module.
filename = _file_info._FileInfo.proto_file_name(
new_attrs.get("__module__", name.lower())
filename = "{0}.proto".format(
new_attrs.get("__module__", name.lower()).replace(".", "/")
)

# Get or create the information about the file, including the
# descriptor to which the new message descriptor shall be added.
file_info = _file_info._FileInfo.maybe_add_descriptor(filename, package)
file_info = _file_info._FileInfo.registry.setdefault(
filename,
_file_info._FileInfo(
descriptor=descriptor_pb2.FileDescriptorProto(
name=filename, package=package, syntax="proto3",
),
enums=collections.OrderedDict(),
messages=collections.OrderedDict(),
name=filename,
nested={},
),
)

# Ensure any imports that would be necessary are assigned to the file
# descriptor proto being created.
Expand All @@ -221,11 +227,6 @@ def __new__(mcls, name, bases, attrs):
for child_path in child_paths:
desc.nested_type.add().MergeFrom(file_info.nested.pop(child_path))

# Same thing, but for enums
child_paths = [p for p in file_info.nested_enum.keys() if local_path == p[:-1]]
for child_path in child_paths:
desc.enum_type.add().MergeFrom(file_info.nested_enum.pop(child_path))

# Add the descriptor to the file if it is a top-level descriptor,
# or to a "holding area" for nested messages otherwise.
if len(local_path) == 1:
Expand Down Expand Up @@ -324,24 +325,17 @@ def deserialize(cls, payload: bytes) -> "Message":
"""
return cls.wrap(cls.pb().FromString(payload))

def to_json(cls, instance, *, use_integers_for_enums=True) -> str:
def to_json(cls, instance) -> str:
"""Given a message instance, serialize it to json
Args:
instance: An instance of this message type, or something
compatible (accepted by the type's constructor).
use_integers_for_enums (Optional(bool)): An option that determines whether enum
values should be represented by strings (False) or integers (True).
Default is True.
Returns:
str: The json string representation of the protocol buffer.
"""
return MessageToJson(
cls.pb(instance),
use_integers_for_enums=use_integers_for_enums,
including_default_value_fields=True,
)
return MessageToJson(cls.pb(instance))

def from_json(cls, payload) -> "Message":
"""Given a json string representing an instance,
Expand Down
29 changes: 0 additions & 29 deletions tests/clam.py
View file

This file was deleted.

22 changes: 0 additions & 22 deletions tests/mollusc.py
View file

This file was deleted.

Loading

0 comments on commit 91c6d7b

Please sign in to comment.