Saved Appengine model: boolean attributes interpreted as None

[mecheverri]

With the fix for this closely-related issue in place, I was able to replicate another bug that I've been seeing while testing. The procedure to reproduce it looks very similar:

1. Using Python 2/legacy Appengine SDK, I ran the following code:

from google.appengine.ext import ndb

class TestInnerModel(ndb.Model):
    s = ndb.StringProperty(required=True, indexed=False)
    b = ndb.BooleanProperty(required=True, default=False)

class TestOuterModel(ndb.Model):
    _use_cache = False
    _use_memcache = False
    imods = ndb.LocalStructuredProperty(TestInnerModel, repeated=True, indexed=False, compressed=True)

OMID = "OMID000001"

im0 = TestInnerModel(s="hi", b=True)
im0.put()
im1 = TestInnerModel()
im1.s = "ho"
im1.put()
im2 = TestInnerModel(s="there")
im2.put()
om0 = TestOuterModel(id=OMID)
om0.imods = [TestInnerModel(s="hi", b=True), TestInnerModel(s="ho"), TestInnerModel(s="there")]
om0.put()

2. Using Python 3.9/google-cloud-sdk, I ran:

from google.cloud import ndb
import redis

def client_context_init():
    return Client().context(
        global_cache=RedisCache(redis.Redis(connection_pool=redis.ConnectionPool())),
        legacy_data=True
    )

class TestInnerModel(ndb.Model):
    s = ndb.StringProperty(required=True)
    b = ndb.BooleanProperty(required=True, default=False)

class TestOuterModel(ndb.Model):
    _use_cache = False
    _use_memcache = False
    imods = ndb.LocalStructuredProperty(TestInnerModel, repeated=True, indexed=False, compressed=True)

OMID = "OMID000001"
with client_context_init():
    print("get model")
    om0 = TestOuterModel.get_by_id(OMID)
    om0.imods.append(TestInnerModel(s="yo"))
    print(om0.to_dict())
    print("save model")
    om0.put()

The latter failed with the following:

Traceback (most recent call last):
  File "<stdin>", line 8, in <module>
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/_options.py", line 89, in wrapper
    return wrapped(*pass_args, **kwargs)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/utils.py", line 114, in wrapper
    return wrapped(*args, **new_kwargs)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/utils.py", line 146, in positional_wrapper
    return wrapped(*args, **kwds)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 5138, in _put
    return self._put_async(_options=kwargs["_options"]).result()
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/tasklets.py", line 191, in result
    self.check_success()
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/tasklets.py", line 138, in check_success
    raise self._exception
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/tasklets.py", line 315, in _advance_tasklet
    yielded = self.generator.send(send_value)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 5198, in put
    ds_entity = _entity_to_ds_entity(self)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 748, in _entity_to_ds_entity
    prop._to_datastore(entity, data)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 4338, in _to_datastore
    keys = super(LocalStructuredProperty, self)._to_datastore(
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 2500, in _to_datastore
    keys = super(BlobProperty, self)._to_datastore(
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 2082, in _to_datastore
    value = self._get_base_value_unwrapped_as_list(entity)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1558, in _get_base_value_unwrapped_as_list
    wrapped = self._get_base_value(entity)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1544, in _get_base_value
    return self._apply_to_values(entity, self._opt_call_to_base_type)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1857, in _apply_to_values
    value[:] = map(function, value)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1619, in _opt_call_to_base_type
    value = _BaseValue(self._call_to_base_type(value))
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1678, in _call_to_base_type
    value = call(value)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 1824, in call
    new_value = method(self, value)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 4281, in _to_base_type
    pb = _entity_to_protobuf(value, set_key=self._keep_keys)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 789, in _entity_to_protobuf
    ds_entity = _entity_to_ds_entity(entity, set_key=set_key)
  File "/vagrant/typhoon/venv/lib/python3.9/site-packages/google/cloud/ndb/model.py", line 752, in _entity_to_ds_entity
    raise exceptions.BadValueError(
google.cloud.ndb.exceptions.BadValueError: Entity has uninitialized properties: b

This is apparently because google-cloud-ndb interpreted the saved TestOuterModel as having null boolean values instead of False, as shown by the print() call:

{'imods': [{'b': True, 's': 'hi'}, {'b': None, 's': 'ho'}, {'b': None, 's': 'there'}, {'b': False, 's': 'yo'}]}

[mecheverri]

A similar effect happens if, in Step 1, you set the s attribute of TestInnerModel to the empty string. In Step 2, that attribute shows up as None and the NDB library raises google.cloud.ndb.exceptions.BadValueError: Entity has uninitialized properties: s. It's probably worth a check to see if other model property types have similar issues.