build: resolve conflicts

Merge branch 'master' into repeated-local-structured-property

google/cloud/ndb/model.py

@@ -294,6 +294,7 @@ class Person(Model):
     "IntegerProperty",
     "FloatProperty",
     "BlobProperty",
+    "CompressedTextProperty",
     "TextProperty",
     "StringProperty",
     "GeoPtProperty",
@@ -2558,6 +2559,129 @@ def _db_set_uncompressed_meaning(self, p):
         raise exceptions.NoLongerImplementedError()
 
 
+class CompressedTextProperty(BlobProperty):
+    """A version of :class:`TextProperty` which compresses values.
+
+    Values are stored as ``zlib`` compressed UTF-8 byte sequences rather than
+    as strings as in a regular :class:`TextProperty`. This class allows NDB to
+    support passing `compressed=True` to :class:`TextProperty`. It is not
+    necessary to instantiate this class directly.
+    """
+
+    __slots__ = ()
+
+    def __init__(self, *args, **kwargs):
+        indexed = kwargs.pop("indexed", False)
+        if indexed:
+            raise NotImplementedError(
+                "A TextProperty cannot be indexed. Previously this was "
+                "allowed, but this usage is no longer supported."
+            )
+
+        kwargs["compressed"] = True
+        super(CompressedTextProperty, self).__init__(*args, **kwargs)
+
+    def _constructor_info(self):
+        """Helper for :meth:`__repr__`.
+
+        Yields:
+            Tuple[str, bool]: Pairs of argument name and a boolean indicating
+            if that argument is a keyword.
+        """
+        parent_init = super(CompressedTextProperty, self).__init__
+        # inspect.signature not available in Python 2.7, so we use positional
+        # decorator combined with argspec instead.
+        argspec = getattr(
+            parent_init, "_argspec", inspect.getargspec(parent_init)
+        )
+        positional = getattr(parent_init, "_positional_args", 1)
+        for index, name in enumerate(argspec.args):
+            if name in ("self", "indexed", "compressed"):
+                continue
+            yield name, index >= positional
+
+    @property
+    def _indexed(self):
+        """bool: Indicates that the property is not indexed."""
+        return False
+
+    def _validate(self, value):
+        """Validate a ``value`` before setting it.
+
+        Args:
+            value (Union[bytes, str]): The value to check.
+
+        Raises:
+            .BadValueError: If ``value`` is :class:`bytes`, but is not a valid
+                UTF-8 encoded string.
+            .BadValueError: If ``value`` is neither :class:`bytes` nor
+                :class:`str`.
+            .BadValueError: If the current property is indexed but the UTF-8
+                encoded value exceeds the maximum length (1500 bytes).
+        """
+        if not isinstance(value, six.text_type):
+            # In Python 2.7, bytes is a synonym for str
+            if isinstance(value, bytes):
+                try:
+                    value = value.decode("utf-8")
+                except UnicodeError:
+                    raise exceptions.BadValueError(
+                        "Expected valid UTF-8, got {!r}".format(value)
+                    )
+            else:
+                raise exceptions.BadValueError(
+                    "Expected string, got {!r}".format(value)
+                )
+
+    def _to_base_type(self, value):
+        """Convert a value to the "base" value type for this property.
+
+        Args:
+            value (Union[bytes, str]): The value to be converted.
+
+        Returns:
+            Optional[bytes]: The converted value. If ``value`` is a
+            :class:`str`, this will return the UTF-8 encoded bytes for it.
+            Otherwise, it will return :data:`None`.
+        """
+        if isinstance(value, six.text_type):
+            return value.encode("utf-8")
+
+    def _from_base_type(self, value):
+        """Convert a value from the "base" value type for this property.
+
+        .. note::
+
+            Older versions of ``ndb`` could write non-UTF-8 ``TEXT``
+            properties. This means that if ``value`` is :class:`bytes`, but is
+            not a valid UTF-8 encoded string, it can't (necessarily) be
+            rejected. But, :meth:`_validate` now rejects such values, so it's
+            not possible to write new non-UTF-8 ``TEXT`` properties.
+
+        Args:
+            value (Union[bytes, str]): The value to be converted.
+
+        Returns:
+            Optional[str]: The converted value. If ``value`` is a valid UTF-8
+                encoded :class:`bytes` string, this will return the decoded
+                :class:`str` corresponding to it. Otherwise, it will return
+                :data:`None`.
+        """
+        if isinstance(value, bytes):
+            try:
+                return value.decode("utf-8")
+            except UnicodeError:
+                pass
+
+    def _db_set_uncompressed_meaning(self, p):
+        """Helper for :meth:`_db_set_value`.
+
+        Raises:
+            NotImplementedError: Always. This method is virtual.
+        """
+        raise NotImplementedError
+
+
 class TextProperty(Property):
     """An unindexed property that contains UTF-8 encoded text values.
 
@@ -2578,10 +2702,37 @@ class Item(ndb.Model):
     .. automethod:: _from_base_type
     .. automethod:: _validate
 
+    Args:
+        name (str): The name of the property.
+        compressed (bool): Indicates if the value should be compressed (via
+            ``zlib``). An instance of :class:`CompressedTextProperty` will be
+            substituted if `True`.
+        indexed (bool): Indicates if the value should be indexed.
+        repeated (bool): Indicates if this property is repeated, i.e. contains
+            multiple values.
+        required (bool): Indicates if this property is required on the given
+            model type.
+        default (Any): The default value for this property.
+        choices (Iterable[Any]): A container of allowed values for this
+            property.
+        validator (Callable[[~google.cloud.ndb.model.Property, Any], bool]): A
+            validator to be used to check values.
+        verbose_name (str): A longer, user-friendly name for this property.
+        write_empty_list (bool): Indicates if an empty list should be written
+            to the datastore.
+
     Raises:
         NotImplementedError: If ``indexed=True`` is provided.
     """
 
+    def __new__(cls, *args, **kwargs):
+        # If "compressed" is True, substitute CompressedTextProperty
+        compressed = kwargs.get("compressed", False)
+        if compressed:
+            return CompressedTextProperty(*args, **kwargs)
+
+        return super(TextProperty, cls).__new__(cls)
+
     def __init__(self, *args, **kwargs):
         indexed = kwargs.pop("indexed", False)
         if indexed:

noxfile.py

@@ -53,7 +53,7 @@ def unit(session):
                 "--cov=tests.unit",
                 "--cov-config",
                 get_path(".coveragerc"),
-                "--cov-report=",
+                "--cov-report=term-missing",
             ]
         )
     run_args.append(get_path("tests", "unit"))

tests/system/test_crud.py

@@ -1200,6 +1200,11 @@ def delete_them(entities):
 def test_insert_entity_with_repeated_local_structured_property_legacy_data(
     client_context, dispose_of, ds_client
 ):
+    """Regression test for #326
+
+    https://github.com/googleapis/python-ndb/issues/326
+    """
+
     class OtherKind(ndb.Model):
         one = ndb.StringProperty()
         two = ndb.StringProperty()
@@ -1228,3 +1233,24 @@ class SomeKind(ndb.Model):
 
         assert isinstance(retrieved.bar[0], OtherKind)
         assert isinstance(retrieved.bar[1], OtherKind)
+
+
+@pytest.mark.usefixtures("client_context")
+def test_compressed_text_property(dispose_of, ds_client):
+    """Regression test for #277
+
+    https://github.com/googleapis/python-ndb/issues/277
+    """
+
+    class SomeKind(ndb.Model):
+        foo = ndb.TextProperty(compressed=True)
+
+    entity = SomeKind(foo="Compress this!")
+    key = entity.put()
+    dispose_of(key._key)
+
+    retrieved = key.get()
+    assert retrieved.foo == "Compress this!"
+
+    ds_entity = ds_client.get(key._key)
+    assert zlib.decompress(ds_entity["foo"]) == b"Compress this!"

tests/unit/test_model.py

@@ -1960,6 +1960,83 @@ class ThisKind(model.Model):
         assert ds_entity["foo"] == [compressed_value_one, compressed_value_two]
 
 
+class TestCompressedTextProperty:
+    @staticmethod
+    def test_constructor_defaults():
+        prop = model.CompressedTextProperty()
+        assert not prop._indexed
+        assert prop._compressed
+
+    @staticmethod
+    def test_constructor_explicit():
+        prop = model.CompressedTextProperty(name="text", indexed=False)
+        assert prop._name == "text"
+        assert not prop._indexed
+
+    @staticmethod
+    def test_constructor_not_allowed():
+        with pytest.raises(NotImplementedError):
+            model.CompressedTextProperty(indexed=True)
+
+    @staticmethod
+    def test_repr():
+        prop = model.CompressedTextProperty(name="text")
+        expected = "CompressedTextProperty('text')"
+        assert repr(prop) == expected
+
+    @staticmethod
+    def test__validate():
+        prop = model.CompressedTextProperty(name="text")
+        assert prop._validate(u"abc") is None
+
+    @staticmethod
+    def test__validate_bad_bytes():
+        prop = model.CompressedTextProperty(name="text")
+        value = b"\x80abc"
+        with pytest.raises(exceptions.BadValueError):
+            prop._validate(value)
+
+    @staticmethod
+    def test__validate_bad_type():
+        prop = model.CompressedTextProperty(name="text")
+        with pytest.raises(exceptions.BadValueError):
+            prop._validate(None)
+
+    @staticmethod
+    def test__to_base_type():
+        prop = model.CompressedTextProperty(name="text")
+        assert prop._to_base_type(b"abc") is None
+
+    @staticmethod
+    def test__to_base_type_converted():
+        prop = model.CompressedTextProperty(name="text")
+        value = b"\xe2\x98\x83"
+        assert prop._to_base_type(u"\N{snowman}") == value
+
+    @staticmethod
+    def test__from_base_type():
+        prop = model.CompressedTextProperty(name="text")
+        assert prop._from_base_type(u"abc") is None
+
+    @staticmethod
+    def test__from_base_type_converted():
+        prop = model.CompressedTextProperty(name="text")
+        value = b"\xe2\x98\x83"
+        assert prop._from_base_type(value) == u"\N{snowman}"
+
+    @staticmethod
+    def test__from_base_type_cannot_convert():
+        prop = model.CompressedTextProperty(name="text")
+        value = b"\x80abc"
+        assert prop._from_base_type(value) is None
+
+    @staticmethod
+    def test__db_set_uncompressed_meaning():
+        prop = model.CompressedTextProperty(name="text")
+        with pytest.raises(NotImplementedError):
+            prop._db_set_uncompressed_meaning(None)
+
+
 class TestTextProperty:
     @staticmethod
     def test_constructor_defaults():
@@ -1977,6 +2054,11 @@ def test_constructor_not_allowed():
         with pytest.raises(NotImplementedError):
             model.TextProperty(indexed=True)
 
+    @staticmethod
+    def test_constructor_compressed():
+        prop = model.TextProperty(compressed=True)
+        assert isinstance(prop, model.CompressedTextProperty)
+
     @staticmethod
     def test_repr():
         prop = model.TextProperty(name="text")