From 0da7b0ce18c1db9ea7b3ef91bd3dc2dd17f1b33a Mon Sep 17 00:00:00 2001 From: Matthias Urban <42069939+urbanmatthias@users.noreply.github.com> Date: Tue, 4 Aug 2020 15:14:27 +0200 Subject: [PATCH] V3.4.0 dev (#452) Python v3.6 is supported again. Namespaces are iterable again. Few bug fixes. --- osp/core/__init__.py | 12 +++- osp/core/ontology/namespace.py | 21 ++++--- osp/core/ontology/namespace_registry.py | 44 ++++++++++----- osp/core/ontology/parser.py | 26 ++++++++- osp/wrappers/simdummy/simdummy_session.py | 31 ++++++---- packageinfo.py | 2 +- tests/parser_test.yml | 2 +- tests/performance_test.py | 2 +- tests/sqlite_performance_test.py | 2 +- tests/test_api_city.py | 2 +- tests/test_dataspace_wrapper.py | 2 +- tests/test_filetransfer.py | 2 +- tests/test_namespace.py | 69 +++++++++++++++++------ tests/test_ontology_entity.py | 2 +- tests/test_parser.py | 7 ++- tests/test_registry_city.py | 2 +- tests/test_session_city.py | 2 +- tests/test_sim_wrapper_city.py | 2 +- tests/test_sqlite_city.py | 2 +- tests/test_transport_auth.py | 2 +- tests/test_transport_session.py | 2 +- tests/test_transport_sim_wrapper.py | 2 +- tests/test_transport_sqlite_city.py | 2 +- tests/test_utils.py | 2 +- tox.ini | 2 +- 25 files changed, 176 insertions(+), 70 deletions(-) diff --git a/osp/core/__init__.py b/osp/core/__init__.py index e97db8c0..82473064 100644 --- a/osp/core/__init__.py +++ b/osp/core/__init__.py @@ -1,5 +1,7 @@ import sys import logging +import osp.core.namespaces +from osp.core.namespaces import _namespace_registry logging.getLogger("rdflib").setLevel(logging.WARNING) @@ -29,8 +31,16 @@ def __getattr__(name): - import osp.core.namespaces if name != "load_tests": logger.warning(f"osp.core.{name} is deprecated. " f"Use osp.core.namespaces.{name} instead.") return getattr(osp.core.namespaces, name) + + +if (sys.version_info.major, sys.version_info.minor) <= (3, 6): + logger.info("We recommend using a python version of at least 3.7") + logger.info(f"osp.core. is deprecated. " + f"Use osp.core.namespaces. instead.") + _namespace_registry.update_namespaces(modules=[ + sys.modules[__name__], namespaces + ]) diff --git a/osp/core/ontology/namespace.py b/osp/core/ontology/namespace.py index 1a3ba215..03644dab 100644 --- a/osp/core/ontology/namespace.py +++ b/osp/core/ontology/namespace.py @@ -129,7 +129,7 @@ def _get(self, name, _case_sensitive=False, _force_by_iri=False): try: return self._do_get(name, _case_sensitive, _force_by_iri) except KeyError as e: - if name.startswith("INVERSE_OF_"): + if name and name.startswith("INVERSE_OF_"): return self._do_get(name[11:], _case_sensitive, _force_by_iri).inverse raise e @@ -196,13 +196,20 @@ def _get_case_insensitive(self, name): f"looked for {alternative} and failed." ) from e - # def __iter__(self): TODO - # """Iterate over the ontology entities in the namespace. + def __iter__(self): + """Iterate over the ontology entities in the namespace. - # :return: An iterator over the entities. - # :rtype: Iterator[OntologyEntity] - # """ - # return iter(self._entities.values()) + :return: An iterator over the entities. + :rtype: Iterator[OntologyEntity] + """ + types = [rdflib.OWL.DatatypeProperty, + rdflib.OWL.ObjectProperty, + rdflib.OWL.Class] + for t in types: + for s, _, _ in self._graph.triples((None, rdflib.RDF.type, t)): + if str(s).startswith(str(self._iri)): + iri_suffix = str(s)[len(str(self._iri)):] + yield self._get(name=None, _force_by_iri=iri_suffix) def __contains__(self, name): return bool(self._get(name)) diff --git a/osp/core/ontology/namespace_registry.py b/osp/core/ontology/namespace_registry.py index 905430fb..832e8e27 100644 --- a/osp/core/ontology/namespace_registry.py +++ b/osp/core/ontology/namespace_registry.py @@ -15,6 +15,8 @@ class NamespaceRegistry(): def __init__(self): + # Do not instantiate you own namespace registry. + # Instead you can use osp.core.namespaces._namespace_registry. self._graph = rdflib.Graph() self._namespaces = dict() @@ -25,7 +27,8 @@ def __iter__(self): :rtype: Iterator[OntologyNamespace] """ for namespace_name in self._namespaces: - yield self._get(namespace_name) + if namespace_name: + yield self._get(namespace_name) def __getattr__(self, name): try: @@ -73,12 +76,16 @@ def _get(self, name): iri=self._namespaces[name]) raise KeyError("Namespace %s not installed." % name) - def update_namespaces(self): + def update_namespaces(self, modules=[]): """Update the namespaces of the namespace registry from the namespaces of the graph.""" self._namespaces = dict() for name, iri in self._graph.namespace_manager.namespaces(): self._namespaces[name.lower()] = iri + for module in modules: + for namespace in self: + setattr(module, namespace.get_name().upper(), namespace) + setattr(module, namespace.get_name().lower(), namespace) def from_iri(self, iri): """Get an entity from IRI. @@ -89,17 +96,28 @@ def from_iri(self, iri): Returns: OntologyEntity: The OntologyEntity. """ - for name, ns_iri in self._graph.namespace_manager.namespaces(): - if str(iri).startswith(str(ns_iri)): - ns = OntologyNamespace( - name=name, - namespace_registry=self, - iri=ns_iri - ) - if ns._reference_by_label: - return ns._get(None, _force_by_iri=iri[len(ns_iri):]) - else: - return ns._get(iri[len(ns_iri):]) + for _name, _ns_iri in self._graph.namespace_manager.namespaces(): + if str(iri).startswith(str(_ns_iri)) and _name: + name = _name + ns_iri = _ns_iri + break + else: + if "#" in str(iri): + ns_iri = str(iri).split("#")[0] + "#" + else: + ns_iri = "/".join(str(iri).split("/")[:-1]) + "/" + name = str(ns_iri) + ns_iri = rdflib.URIRef(ns_iri) + + ns = OntologyNamespace( + name=name, + namespace_registry=self, + iri=ns_iri + ) + if ns._reference_by_label: + return ns._get(None, _force_by_iri=iri[len(ns_iri):]) + else: + return ns._get(iri[len(ns_iri):]) def clear(self): """Clear the loaded Graph and load cuba only. diff --git a/osp/core/ontology/parser.py b/osp/core/ontology/parser.py index 9a5dccb3..3ed13508 100644 --- a/osp/core/ontology/parser.py +++ b/osp/core/ontology/parser.py @@ -224,6 +224,7 @@ def _parse_rdf(self, **kwargs): default_rels[iri] = default_rel reference_styles[iri] = reference_style + self._check_namespaces() self._add_cuba_triples(active_rels) self._add_default_rel_triples(default_rels) self._add_reference_style_triples(reference_styles) @@ -252,8 +253,14 @@ def _add_cuba_triples(self, active_rels): active_rels (List[str]): The URIs of the active relationships. """ for rel in active_rels: + iri = rdflib.URIRef(rel) + if (iri, rdflib.RDF.type, rdflib.OWL.ObjectProperty) \ + not in self.graph: + raise ValueError(f"Specified relationship {rel} as " + f"active relationship, which is not " + f"a valid object property in the ontology.") self.graph.add( - (rdflib.URIRef(rel), rdflib.RDFS.subPropertyOf, + (iri, rdflib.RDFS.subPropertyOf, rdflib_cuba.activeRelationship) ) @@ -271,3 +278,20 @@ def _add_reference_style_triples(self, reference_styles): rdflib_cuba._reference_by_label, rdflib.Literal(True) )) + + def _check_namespaces(self): + namespaces = set(x for _, x in self.graph.namespaces() + if not x.startswith("http://www.w3.org/")) + for s, p, o in self.graph: + pop = None + for ns in namespaces: + if s.startswith(ns): + pop = ns + logger.debug(f"There exists an entity for namespace {ns}:" + f"\n\t{s, p, o}") + if pop: + namespaces.remove(pop) + if not namespaces: + break + for namespace in namespaces: + logger.warning(f"There exists no entity for namespace {namespace}") diff --git a/osp/wrappers/simdummy/simdummy_session.py b/osp/wrappers/simdummy/simdummy_session.py index ff95bf69..90a688e9 100644 --- a/osp/wrappers/simdummy/simdummy_session.py +++ b/osp/wrappers/simdummy/simdummy_session.py @@ -4,12 +4,19 @@ from osp.core.session.sim_wrapper_session import SimWrapperSession from osp.core.utils import change_oclass +try: + from osp.core.namespaces import city +except ImportError: + from osp.core.ontology import Parser + from osp.core.namespaces import _namespace_registry + Parser(_namespace_registry._graph).parse("city") + _namespace_registry.update_namespaces() + city = _namespace_registry.city + class SimDummySession(SimWrapperSession): def __init__(self, **kwargs): super().__init__(engine=DummySyntacticLayer(), **kwargs) - from osp.core.namespaces import city - self.onto = city self._person_map = dict() def __str__(self): @@ -24,7 +31,7 @@ def _load_from_backend(self, uids, expired=None): # update the age of each person and delete persons that became citizens for uid in uids: root_cuds_object = self._registry.get(self.root) - cities = root_cuds_object.get(oclass=self.onto.City) + cities = root_cuds_object.get(oclass=city.City) if uid == self.root: yield self._load_wrapper(uid) elif cities and uid == cities[0].uid: @@ -41,7 +48,7 @@ def _load_person(self, uid): person = self._registry.get(uid) idx = self._person_map[uid] person.age = self._engine.get_person(idx)[1].age - if person.is_a(self.onto.Citizen): + if person.is_a(city.Citizen): return person self._check_convert_to_inhabitant(uid) return person @@ -49,7 +56,7 @@ def _load_person(self, uid): def _load_city(self, uid): c = self._registry.get(uid) inhabitant_uids = set( - [x.uid for x in c.get(rel=self.onto.hasInhabitant)] + [x.uid for x in c.get(rel=city.hasInhabitant)] ) person_uids = self._person_map.keys() - inhabitant_uids for person_uid in person_uids: @@ -58,22 +65,22 @@ def _load_city(self, uid): def _load_wrapper(self, uid): wrapper = self._registry.get(uid) - for person in wrapper.get(oclass=self.onto.Person): + for person in wrapper.get(oclass=city.Person): self.refresh(person.uid) return wrapper def _check_convert_to_inhabitant(self, uid): wrapper = self._registry.get(self.root) - c = wrapper.get(oclass=self.onto.City)[0] + c = wrapper.get(oclass=city.City)[0] idx = self._person_map[uid] is_inhabitant, dummy_person = self._engine.get_person(idx) if is_inhabitant: person = self._registry.get(uid) - change_oclass(person, self.onto.Citizen, + change_oclass(person, city.Citizen, {"name": dummy_person.name, "age": dummy_person.age}) - wrapper.remove(person, rel=self.onto.hasPart) - c.add(person, rel=self.onto.hasInhabitant) + wrapper.remove(person, rel=city.hasPart) + c.add(person, rel=city.hasInhabitant) # OVERRIDE def _apply_added(self, root_obj, buffer): @@ -85,9 +92,9 @@ def _apply_added(self, root_obj, buffer): key=lambda x: x.name if hasattr(x, "name") else "0") for added in sorted_added: if ( - added.is_a(self.onto.Person) + added.is_a(city.Person) and self.root in map(lambda x: x.uid, - added.get(rel=self.onto.isPartOf)) + added.get(rel=city.isPartOf)) ): idx = self._engine.add_person(DummyPerson(added.name, added.age)) diff --git a/packageinfo.py b/packageinfo.py index f68cecba..a93d09e4 100644 --- a/packageinfo.py +++ b/packageinfo.py @@ -1,2 +1,2 @@ NAME = "osp-core" -VERSION = "3.3.9" +VERSION = "3.4.0" diff --git a/tests/parser_test.yml b/tests/parser_test.yml index 4c52efbe..923ebfff 100644 --- a/tests/parser_test.yml +++ b/tests/parser_test.yml @@ -6,5 +6,5 @@ ontology_file: parser_test.ttl format: "ttl" default_relationship: http://www.osp-core.com/parser_test#relationshipA active_relationships: - - http://www.osp-core.com/parser_test#relationshipA, + - http://www.osp-core.com/parser_test#relationshipA - http://www.osp-core.com/parser_test#relationshipB diff --git a/tests/performance_test.py b/tests/performance_test.py index f233a164..9539a5e4 100644 --- a/tests/performance_test.py +++ b/tests/performance_test.py @@ -12,7 +12,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city RUN_PERFORMANCE_TEST = False diff --git a/tests/sqlite_performance_test.py b/tests/sqlite_performance_test.py index fecff90a..6361f81d 100644 --- a/tests/sqlite_performance_test.py +++ b/tests/sqlite_performance_test.py @@ -20,7 +20,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city RUN_PERFORMANCE_TEST = False DB = "performance_test.db" diff --git a/tests/test_api_city.py b/tests/test_api_city.py index ef5889f3..027e6195 100644 --- a/tests/test_api_city.py +++ b/tests/test_api_city.py @@ -14,7 +14,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city class TestAPICity(unittest.TestCase): diff --git a/tests/test_dataspace_wrapper.py b/tests/test_dataspace_wrapper.py index a46a706e..a949bc87 100644 --- a/tests/test_dataspace_wrapper.py +++ b/tests/test_dataspace_wrapper.py @@ -24,7 +24,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city HOST = "127.0.0.1" PORT = 8681 diff --git a/tests/test_filetransfer.py b/tests/test_filetransfer.py index 5a96909b..d60fc332 100644 --- a/tests/test_filetransfer.py +++ b/tests/test_filetransfer.py @@ -38,7 +38,7 @@ def async_test(x): from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city HOST = "127.0.0.1" PORT = 8645 diff --git a/tests/test_namespace.py b/tests/test_namespace.py index 8aa54e4c..87b4c92c 100644 --- a/tests/test_namespace.py +++ b/tests/test_namespace.py @@ -113,24 +113,50 @@ def test_namespace_registry_from_iri(self): self.assertEqual(r.namespace.get_name(), "city") self.assertEqual(r.name, "hasPart") from osp.core.namespaces import from_iri - c = from_iri(rdflib_cuba.Class) - self.assertIsInstance(c, OntologyClass) - self.assertEqual(c.namespace.get_name(), "cuba") - self.assertEqual(c.name, "Class") + import osp.core.namespaces + old_ns_reg = osp.core.namespaces._namespace_registry + try: + osp.core.namespaces._namespace_registry = self.namespace_registry + c = from_iri(rdflib_cuba.Class) + self.assertIsInstance(c, OntologyClass) + self.assertEqual(c.namespace.get_name(), "cuba") + self.assertEqual(c.name, "Class") - self.graph.add(( - ns_iri, - rdflib_cuba._reference_by_label, - rdflib.Literal(True) - )) - c = self.namespace_registry.from_iri(city_iri) - self.assertIsInstance(c, OntologyClass) - self.assertEqual(c.namespace.get_name(), "city") - self.assertEqual(c.name, "City_T") - r = self.namespace_registry.from_iri(hasPart_iri) - self.assertIsInstance(r, OntologyRelationship) - self.assertEqual(r.namespace.get_name(), "city") - self.assertEqual(r.name, "hasPart_T") + self.graph.add(( + ns_iri, + rdflib_cuba._reference_by_label, + rdflib.Literal(True) + )) + c = self.namespace_registry.from_iri(city_iri) + self.assertIsInstance(c, OntologyClass) + self.assertEqual(c.namespace.get_name(), "city") + self.assertEqual(c.name, "City_T") + r = self.namespace_registry.from_iri(hasPart_iri) + self.assertIsInstance(r, OntologyRelationship) + self.assertEqual(r.namespace.get_name(), "city") + self.assertEqual(r.name, "hasPart_T") + + # undefined namespace + self.graph.add(( + rdflib.URIRef("a/b#c"), + rdflib.RDF.type, + rdflib.OWL.Class + )) + self.graph.add(( + rdflib.URIRef("d/e/f"), + rdflib.RDF.type, + rdflib.OWL.Class + )) + a = from_iri("a/b#c") + b = from_iri("d/e/f") + self.assertIsInstance(a, OntologyClass) + self.assertEqual(a.namespace.get_name(), "a/b#") + self.assertEqual(a.name, "c") + self.assertIsInstance(b, OntologyClass) + self.assertEqual(b.namespace.get_name(), "d/e/") + self.assertEqual(b.name, "f") + finally: + osp.core.namespaces._namespace_registry = old_ns_reg def test_namespace_registry_update_namespaces(self): self.graph.bind("a", rdflib.URIRef("aaa")) @@ -296,6 +322,15 @@ def test_contains(self): self.assertIn("City", namespace) self.assertIn("hasPart", namespace) + def test_iter(self): + self.installer.install("city") + namespace = self.namespace_registry.city + entities = set(namespace) + self.assertIn(namespace.encloses, entities) + self.assertIn(namespace.City, entities) + self.assertIn(namespace.name, entities) + self.assertEqual(len(entities), 32) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_ontology_entity.py b/tests/test_ontology_entity.py index f6550142..221327c7 100644 --- a/tests/test_ontology_entity.py +++ b/tests/test_ontology_entity.py @@ -12,7 +12,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city class TestOntologyEntity(unittest.TestCase): diff --git a/tests/test_parser.py b/tests/test_parser.py index 545861d4..f3167afe 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -25,10 +25,15 @@ def setUp(self): self.parser = Parser(self.graph) def test_add_cuba_triples(self): + self.graph.add((rdflib.URIRef("has_part"), rdflib.RDF.type, + rdflib.OWL.ObjectProperty)) + self.graph.add((rdflib.URIRef("encloses"), rdflib.RDF.type, + rdflib.OWL.ObjectProperty)) + pre = set(self.graph) self.parser._add_cuba_triples([ rdflib.URIRef("has_part"), rdflib.URIRef("encloses") ]) - self.assertEqual(set(self.graph), { + self.assertEqual(set(self.graph) - pre, { (rdflib.URIRef("encloses"), rdflib.RDFS.subPropertyOf, rdflib_cuba.activeRelationship), (rdflib.URIRef("has_part"), rdflib.RDFS.subPropertyOf, diff --git a/tests/test_registry_city.py b/tests/test_registry_city.py index b6ced84e..ebd60298 100644 --- a/tests/test_registry_city.py +++ b/tests/test_registry_city.py @@ -10,7 +10,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city class TestRegistryCity(unittest.TestCase): diff --git a/tests/test_session_city.py b/tests/test_session_city.py index 4209f389..19e358a3 100644 --- a/tests/test_session_city.py +++ b/tests/test_session_city.py @@ -11,7 +11,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city class TestSessionCity(unittest.TestCase): diff --git a/tests/test_sim_wrapper_city.py b/tests/test_sim_wrapper_city.py index 42998be4..6b75dff9 100644 --- a/tests/test_sim_wrapper_city.py +++ b/tests/test_sim_wrapper_city.py @@ -12,7 +12,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city class TestSimWrapperCity(unittest.TestCase): diff --git a/tests/test_sqlite_city.py b/tests/test_sqlite_city.py index 02dc4a11..e7550790 100644 --- a/tests/test_sqlite_city.py +++ b/tests/test_sqlite_city.py @@ -11,7 +11,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city DB = "test_sqlite.db" diff --git a/tests/test_transport_auth.py b/tests/test_transport_auth.py index 89b4aa37..dd85f9ed 100644 --- a/tests/test_transport_auth.py +++ b/tests/test_transport_auth.py @@ -16,7 +16,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city HOST = "127.0.0.1" PORT1 = 8469 diff --git a/tests/test_transport_session.py b/tests/test_transport_session.py index 3cef514d..a167f967 100644 --- a/tests/test_transport_session.py +++ b/tests/test_transport_session.py @@ -24,7 +24,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city CUDS_DICT = { "oclass": "city.Citizen", diff --git a/tests/test_transport_sim_wrapper.py b/tests/test_transport_sim_wrapper.py index 35febde7..a390077c 100644 --- a/tests/test_transport_sim_wrapper.py +++ b/tests/test_transport_sim_wrapper.py @@ -18,7 +18,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city HOST = "127.0.0.1" PORT = 8689 diff --git a/tests/test_transport_sqlite_city.py b/tests/test_transport_sqlite_city.py index 64ed80fa..ee73bf84 100644 --- a/tests/test_transport_sqlite_city.py +++ b/tests/test_transport_sqlite_city.py @@ -21,7 +21,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city HOST = "127.0.0.1" PORT = 8687 diff --git a/tests/test_utils.py b/tests/test_utils.py index ae3a6931..020e3db3 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -30,7 +30,7 @@ from osp.core.namespaces import _namespace_registry Parser(_namespace_registry._graph).parse("city") _namespace_registry.update_namespaces() - from osp.core.namespaces import city + city = _namespace_registry.city CUDS_DICT = { "oclass": "city.Citizen", diff --git a/tox.ini b/tox.ini index 8401dc5d..3690fd4f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py37,py38 +envlist = py36, py37,py38 [testenv] deps =