Skip to content

Commit

Permalink
fix(zlib): Accomodate different Zlib compression levels (#852)
Browse files Browse the repository at this point in the history
Different Zlib compression levels produce different
compression markers.

Co-authored-by: Zhou Wang <wangzhou@google.com>

Co-authored-by: Zhou Wang <wangzhou@google.com>
  • Loading branch information
rwhogg and Zhou Wang authored Dec 12, 2022
1 parent d2a3366 commit c1ab83b
Showing 1 changed file with 11 additions and 7 deletions.
18 changes: 11 additions & 7 deletions google/cloud/ndb/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,10 +334,14 @@ class Person(Model):
_MEANING_PREDEFINED_ENTITY_USER = 20
_MEANING_COMPRESSED = 22

# As produced by zlib. Indicates compressed byte sequence using DEFLATE at
# default compression level, with a 32K window size.
# From https://github.com/madler/zlib/blob/master/doc/rfc1950.txt
_ZLIB_COMPRESSION_MARKER = b"x\x9c"
_ZLIB_COMPRESSION_MARKERS = (
# As produced by zlib. Indicates compressed byte sequence using DEFLATE at
# default compression level, with a 32K window size.
# From https://github.com/madler/zlib/blob/master/doc/rfc1950.txt
b"x\x9c",
# Other compression levels produce the following marker.
b"x^",
)

_MAX_STRING_LENGTH = 1500
Key = key_module.Key
Expand Down Expand Up @@ -2619,7 +2623,7 @@ def _from_base_type(self, value):
return

if self._compressed and not isinstance(value, _CompressedValue):
if not value.startswith(_ZLIB_COMPRESSION_MARKER):
if not value.startswith(_ZLIB_COMPRESSION_MARKERS):
return value
value = _CompressedValue(value)

Expand All @@ -2645,13 +2649,13 @@ def _to_datastore(self, entity, data, prefix="", repeated=False):
if self._repeated:
compressed_value = []
for rval in value:
if rval and not rval.startswith(_ZLIB_COMPRESSION_MARKER):
if rval and not rval.startswith(_ZLIB_COMPRESSION_MARKERS):
rval = zlib.compress(rval)
compressed_value.append(rval)
value = compressed_value
data[key] = value
if not self._repeated:
if value and not value.startswith(_ZLIB_COMPRESSION_MARKER):
if value and not value.startswith(_ZLIB_COMPRESSION_MARKERS):
value = zlib.compress(value)
data[key] = value

Expand Down

0 comments on commit c1ab83b

Please sign in to comment.