From a7875fc174d565529a7b201cad8dec16f958f9ad Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Thu, 14 Mar 2019 10:06:47 +0200 Subject: [PATCH 1/7] Let KeyProperty know about the namespace too --- .gitignore | 2 ++ ndb_orm/model.py | 14 +++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2790314..8e74eb8 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,5 @@ pip-log.txt .tox nosetests.xml +# PyEnv +.python-version diff --git a/ndb_orm/model.py b/ndb_orm/model.py index 6f6cba0..fd14eb9 100644 --- a/ndb_orm/model.py +++ b/ndb_orm/model.py @@ -1995,7 +1995,11 @@ def _db_get_value(self, v): path_flat = [] for elm in v.key_value.path: path_flat.extend([elm.kind, elm.name if elm.name != '' else elm.id]) - return key_module.Key(*path_flat, project=v.key_value.partition_id.project_id) + return key_module.Key( + *path_flat, + project=v.key_value.partition_id.project_id, + namespace=v.key_value.partition_id.namespace_id + ) class BlobKeyProperty(Property): @@ -2077,7 +2081,7 @@ def _db_set_value(self, v, value): # raise NotImplementedError('DatetimeProperty %s can only support UTC. ' # 'Please derive a new Property to support ' # 'alternative timezones.' % self._name) - + v.timestamp_value.CopyFrom(helpers.datetime_to_pb_timestamp(value)) # TODO old style ndb write - not possible anymore ! @@ -2913,7 +2917,7 @@ def __init__(*args, **kwds): self._key = _validate_key(key, entity=self) # elif (id is not None or parent is not None or # project is not None or namespace is not None): - + else: path = [id] if id else [] # path self._key = key_module.Key( @@ -3191,14 +3195,14 @@ def _from_pb(cls, pb, set_key=True, ent=None, key=None): # iterate over properties projection = [] for name, p in six.iteritems(pb.properties): - # TODO + # TODO if p.meaning == PROPERTY_INDEX_VALUE: projection.append(name) indexed = not p.exclude_from_indexes prop = ent._get_property_for(name, p, indexed) prop._deserialize(name, ent, p) - # TODO + # TODO # ent._set_projection(projection) return ent From 3bd382b37fdfb9058085bda3ec16f743987a6a65 Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Thu, 14 Mar 2019 10:50:36 +0200 Subject: [PATCH 2/7] Support `get` and `delete` over Keys --- ndb_orm/__init__.py | 25 +++++++++++++++++++++++-- ndb_orm/key.py | 2 +- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ndb_orm/__init__.py b/ndb_orm/__init__.py index 7c0937d..3d469d2 100644 --- a/ndb_orm/__init__.py +++ b/ndb_orm/__init__.py @@ -65,7 +65,7 @@ real_entity_from_protobuf = None real_entity_to_protobuf = None -def enable_use_with_gcd(project=None, namespace=None): +def enable_use_with_gcd(project=None, namespace=None, client=None): from google.cloud import datastore from google.cloud.datastore.key import Key as DatastoreKey from google.cloud.datastore_v1.proto import entity_pb2 @@ -142,7 +142,28 @@ def new_entity_to_protobuf(entity): datastore.helpers.entity_from_protobuf = real_entity_from_protobuf datastore.helpers.entity_to_protobuf = real_entity_to_protobuf - key_module.KeyBase = DatastoreKey + class KeyBase(DatastoreKey): + + """Custom Key class that implements missing legacy methods.""" + + @property + def client(self): + if client: + return client + raise NotImplementedError( + "`KeyBase` class doesn't have a client associated with it; " + "please provide a `client` too when enabling with GCD" + ) + + def get(self): + """Get the entity object by using the client with its key.""" + return self.client.get(self) + + def delete(self): + """Remove the entity object by using the client with its key.""" + self.client.delete(self) + + key_module.KeyBase = KeyBase entity_module.Entity = entity_pb2.Entity entity_module.Property = entity_module.Property entity_module.Reference = entity_module.Reference diff --git a/ndb_orm/key.py b/ndb_orm/key.py index 6f8d847..b9ae71a 100644 --- a/ndb_orm/key.py +++ b/ndb_orm/key.py @@ -45,7 +45,7 @@ def __call__(self, model_cls, *path_args, **kwargs): # accept both, plain strings and object instances as path if not isinstance(model_cls, six.string_types): path_args[i] = path_args[i]._get_kind() - + return KeyBase( model_cls_str, *path_args, From 0af23842d54c94ca8b816e5506d0a04e6580f011 Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Thu, 14 Mar 2019 12:24:27 +0200 Subject: [PATCH 3/7] Better namespacing logic on keys --- ndb_orm/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndb_orm/model.py b/ndb_orm/model.py index fd14eb9..a5d180d 100644 --- a/ndb_orm/model.py +++ b/ndb_orm/model.py @@ -1998,7 +1998,7 @@ def _db_get_value(self, v): return key_module.Key( *path_flat, project=v.key_value.partition_id.project_id, - namespace=v.key_value.partition_id.namespace_id + namespace=v.key_value.partition_id.namespace_id or None ) From d2fd217edf9a1131e3296aa0b7bda0e3f0c5dbc4 Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Thu, 14 Mar 2019 13:00:03 +0200 Subject: [PATCH 4/7] Fix default namespace argument shadowing issue --- ndb_orm/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ndb_orm/__init__.py b/ndb_orm/__init__.py index 3d469d2..c1766d8 100644 --- a/ndb_orm/__init__.py +++ b/ndb_orm/__init__.py @@ -95,8 +95,9 @@ def model_from_protobuf_datastore(pb): entity.key = key return entity - def model_to_protobuf_datastore(entity_of_ndb_model, project, namespace=None): - if namespace and entity_of_ndb_model._key and (entity_of_ndb_model._key.namespace == None): + def model_to_protobuf_datastore(entity_of_ndb_model, project, namespace=namespace): + if namespace and entity_of_ndb_model._key and ( + entity_of_ndb_model._key.namespace is None): # add namespace entity_of_ndb_model._key._namespace = namespace entity_of_ndb_model._prepare_for_put() From 5967e6e8ea0305c5be2ed6ae900149bcab751585 Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Thu, 14 Mar 2019 13:16:43 +0200 Subject: [PATCH 5/7] Add Key and KeyProperty tests --- tests/tests.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index d676e62..9edfc54 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # vim: sts=2:ts=2:sw=2 +import unittest from unittest import TestCase, main import os @@ -412,5 +413,46 @@ def test_repeated_structuredproperty(self): self.assertEqual(foo_recovered.a[2].b.d, 3) +class KeyDepartment(Department): + + dep_key = ndb.KeyProperty(kind=Department, required=True) + + +@unittest.skipIf(not USE_DATASTORE, "not supported without Datastore emulator") +class TestEntityKey(TestCase): + + NAMESPACE = "test-namespace" + + @classmethod + def setUpClass(cls): + from google.cloud import datastore + from .gcloud_credentials import EmulatorCredentials + cls.client = datastore.Client(project=PROJECT, credentials=EmulatorCredentials()) + ndb.enable_use_with_gcd(project=PROJECT, namespace=cls.NAMESPACE, client=cls.client) + + def setUp(self): + self._dep = Department(name="test_dep") + self.client.put(self._dep) + self._key_dep = KeyDepartment(name="test_key_dep", dep_key=self._dep.key) + self.client.put(self._key_dep) + + def tearDown(self): + self.client.delete(self._dep.key) + self.client.delete(self._key_dep.key) + + def test_retrieval(self): + self.assertEqual("test_key_dep", self._key_dep.key.get().name) + + def test_removal(self): + self.assertTrue(self._key_dep.key.get()) + self._key_dep.key.delete() + self.assertFalse(self._key_dep.key.get()) + + def test_keyprop_namespace(self): + key = self._key_dep.key.get().dep_key + self.assertEqual(self.NAMESPACE, key.namespace) + self.assertEqual("test_dep", key.get().name) + + if __name__ == '__main__': main() From 97eb1009a97a932a652bcd98b260242103bbfc50 Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Sun, 21 Apr 2019 14:08:23 +0300 Subject: [PATCH 6/7] Use our Key class with entities's keys too --- ndb_orm/__init__.py | 8 +++++++- tests/tests.py | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ndb_orm/__init__.py b/ndb_orm/__init__.py index c1766d8..5392dc2 100644 --- a/ndb_orm/__init__.py +++ b/ndb_orm/__init__.py @@ -92,7 +92,13 @@ def model_from_protobuf_datastore(pb): return None entity = modelclass._from_pb(pb, key=key, set_key=False) #entity = modelclass._from_pb(pb, key=key, set_key=True) - entity.key = key + # NOTE(cmiN): Make sure we use the same augmented custom Key class. + entity.key = Key( + *key._flat_path, + parent=key._parent, + namespace=key._namespace, + project=key._project, + ) return entity def model_to_protobuf_datastore(entity_of_ndb_model, project, namespace=namespace): diff --git a/tests/tests.py b/tests/tests.py index 9edfc54..140cc42 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -7,6 +7,7 @@ import os import binascii import ndb_orm as ndb +from ndb_orm import key_module from protorpc import messages from google.cloud.datastore_v1.proto import entity_pb2 from . import person_pb2 @@ -443,6 +444,14 @@ def tearDown(self): def test_retrieval(self): self.assertEqual("test_key_dep", self._key_dep.key.get().name) + def test_self_retrieval(self): + dep = self._dep.key.get() + self.assertEqual("test_dep", dep.name) + self_key = dep.key + self.assertIsInstance(self_key, key_module.KeyBase) + self_dep = self_key.get() + self.assertEqual("test_dep", self_dep.name) + def test_removal(self): self.assertTrue(self._key_dep.key.get()) self._key_dep.key.delete() From bf3bd50e85feb7f4cd57c47b48b83ece4b874cca Mon Sep 17 00:00:00 2001 From: Cosmin Poieana Date: Sun, 21 Apr 2019 14:51:19 +0300 Subject: [PATCH 7/7] Python2 compatible instantiation --- ndb_orm/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ndb_orm/__init__.py b/ndb_orm/__init__.py index 5392dc2..3143a65 100644 --- a/ndb_orm/__init__.py +++ b/ndb_orm/__init__.py @@ -97,7 +97,7 @@ def model_from_protobuf_datastore(pb): *key._flat_path, parent=key._parent, namespace=key._namespace, - project=key._project, + project=key._project ) return entity