From cf4b17ddb7a81250b3e589f6473105b5db867683 Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Sat, 6 Feb 2021 18:53:23 +0100 Subject: [PATCH 1/7] Add DataBindingMeta metaclass and DataBindingConverter --- CHANGELOG.rst | 7 +++++++ doc/conf.py | 4 ++-- doc/extras.rst | 5 ++--- setup.py | 2 +- tests/test_dataobjects.py | 31 ++++++++++++++++++++++++++++- xmlschema/__init__.py | 2 +- xmlschema/converters/dataelement.py | 27 ++++++++++++++++++++++++- xmlschema/dataobjects.py | 19 +++++++++++------- xmlschema/validators/elements.py | 11 +++++++++- xmlschema/validators/global_maps.py | 25 +++++++++++++---------- xmlschema/validators/schema.py | 5 +++++ 11 files changed, 110 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2dbf76ac..e9c2eef8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ CHANGELOG ********* +`v1.6.0`_ (2021-02-05) +====================== +* TODO: Add DataElementMeta metaclass for customizing DataElement subclasses +* TODO: Implement PythonGenerator with three set of templates (xmlschema API + based, simple class hierarchy, dataclass hierarchy). + `v1.5.0`_ (2021-02-05) ====================== * Add DataElement class for creating objects with schema bindings @@ -402,3 +408,4 @@ v0.9.6 (2017-05-05) .. _v1.4.1: https://github.com/brunato/xmlschema/compare/v1.4.0...v1.4.1 .. _v1.4.2: https://github.com/brunato/xmlschema/compare/v1.4.1...v1.4.2 .. _v1.5.0: https://github.com/brunato/xmlschema/compare/v1.4.2...v1.5.0 +.. _v1.6.0: https://github.com/brunato/xmlschema/compare/v1.5.0...v1.6.0 diff --git a/doc/conf.py b/doc/conf.py index 2708811e..262e0d35 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -63,9 +63,9 @@ # built documents. # # The short X.Y version. -version = '1.5' +version = '1.6' # The full version, including alpha/beta/rc tags. -release = '1.5.0' +release = '1.6.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/extras.rst b/doc/extras.rst index c16452b7..9a59fdf6 100644 --- a/doc/extras.rst +++ b/doc/extras.rst @@ -2,9 +2,8 @@ Extra features ************** -The subpackage *xmlschema.extras* contThe subpackage *xmlschema.extras* -acts as a container of a set of extra modules or subpackages that can be -useful for specific needs. +The subpackage *xmlschema.extras* acts as a container of a set of extra +modules or subpackages that can be useful for specific needs. These codes are not imported during normal library usage and may require additional dependencies to be installed. This choice should be facilitate diff --git a/setup.py b/setup.py index f51b84a7..378bf8a1 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name='xmlschema', - version='1.5.0', + version='1.6.0', packages=find_packages(include=['xmlschema', 'xmlschema.*']), include_package_data=True, entry_points={ diff --git a/tests/test_dataobjects.py b/tests/test_dataobjects.py index 2a9d58c3..f63291be 100644 --- a/tests/test_dataobjects.py +++ b/tests/test_dataobjects.py @@ -19,7 +19,7 @@ from xmlschema import XMLSchema, fetch_namespaces, etree_tostring from xmlschema.helpers import is_etree_element -from xmlschema.dataobjects import DataElement +from xmlschema.dataobjects import DataElement, DataBindingMeta from xmlschema.testing import etree_elements_assert_equal from xmlschema.converters import DataElementConverter @@ -216,6 +216,35 @@ def test_serialize_to_xml_source(self): self.assertTrue(xml_source.endswith('')) +class TestDataElementMeta(TestDataElement): + + def test_data_element_metaclass(self): + xsd_element = self.col_schema.elements['collection'] + collection_class = DataBindingMeta(xsd_element, (DataElement,), {}) + self.assertEqual(collection_class.__name__, 'CollectionElement') + self.assertEqual(collection_class.__qualname__, 'CollectionElement') + self.assertIsNone(collection_class.__module__) + self.assertEqual(collection_class.namespace, 'http://example.com/ns/collection') + self.assertEqual(collection_class.xsd_version, '1.0') + + def test_element_binding(self): + xsd_element = self.col_schema.elements['collection'] + xsd_element.binding = None + + try: + cls = xsd_element.create_binding() + self.assertIsNot(xsd_element.binding, DataElement) + self.assertTrue(issubclass(xsd_element.binding, DataElement)) + self.assertIsInstance(xsd_element.binding, DataBindingMeta) + self.assertIs(cls, xsd_element.binding) + finally: + xsd_element.binding = None + + def test_schema_bindings(self): + schema = XMLSchema(self.col_xsd_filename) + schema.maps.create_bindings() + + if __name__ == '__main__': import platform header_template = "Test xmlschema data objects with Python {} on {}" diff --git a/xmlschema/__init__.py b/xmlschema/__init__.py index 2415bad3..5d69eb4d 100644 --- a/xmlschema/__init__.py +++ b/xmlschema/__init__.py @@ -30,7 +30,7 @@ XsdComponent, XsdType, XsdElement, XsdAttribute ) -__version__ = '1.5.0' +__version__ = '1.6.0' __author__ = "Davide Brunato" __contact__ = "brunato@sissa.it" __copyright__ = "Copyright 2016-2021, SISSA" diff --git a/xmlschema/converters/dataelement.py b/xmlschema/converters/dataelement.py index 93e0830c..37a53e12 100644 --- a/xmlschema/converters/dataelement.py +++ b/xmlschema/converters/dataelement.py @@ -8,7 +8,7 @@ # @author Davide Brunato # from ..exceptions import XMLSchemaValueError -from ..dataobjects import ElementData, DataElement +from ..dataobjects import ElementData, DataElement, DataBindingMeta from .default import XMLSchemaConverter @@ -77,3 +77,28 @@ def element_encode(self, data_element, xsd_element, level=0): else (next(cdata_num), e) for e in data_element ] return ElementData(data_element.tag, None, content, attributes) + + +class DataBindingConverter(DataElementConverter): + """ + XML Schema based converter class for DataElementMeta metaclass objects. + + """ + def element_decode(self, data, xsd_element, xsd_type=None, level=0): + cls = xsd_element.binding or xsd_element.create_binding(self.data_element_class) + data_element = cls( + tag=data.tag, + value=data.text, + nsmap=self.namespaces, + xsd_element=xsd_element, + xsd_type=xsd_type + ) + data_element.attrib.update((k, v) for k, v in self.map_attributes(data.attributes)) + + if (xsd_type or xsd_element.type).model_group is not None: + data_element.extend([ + value if value is not None else self.list([name]) + for name, value, _ in self.map_content(data.content) + ]) + + return data_element diff --git a/xmlschema/dataobjects.py b/xmlschema/dataobjects.py index bddc4c2d..a21ba5a8 100644 --- a/xmlschema/dataobjects.py +++ b/xmlschema/dataobjects.py @@ -208,10 +208,15 @@ def iterchildren(self, tag=None): yield child -class DataElementMeta(ABCMeta): - """ - TODO: A metaclass for defining derived data element classes. - The underlining idea is to develop schema API for XSD elements - that can be generated by option and stored in a registry if - necessary. - """ +class DataBindingMeta(ABCMeta): + """Metaclass for creating classes with bindings to XSD elements.""" + def __new__(mcs, xsd_element, bases, attrs): + name = xsd_element.local_name + class_name = '{}Element'.format(name.replace('_', '').title()) + + attrs['__module__'] = None + attrs['namespace'] = xsd_element.target_namespace + attrs['xsd_version'] = xsd_element.xsd_version + cls = super(DataBindingMeta, mcs).__new__(mcs, class_name, bases, attrs) + return cls + diff --git a/xmlschema/validators/elements.py b/xmlschema/validators/elements.py index 5efa47e2..464ab84e 100644 --- a/xmlschema/validators/elements.py +++ b/xmlschema/validators/elements.py @@ -23,7 +23,7 @@ from ..etree import etree_element from ..helpers import get_qname, get_namespace, etree_iter_location_hints, \ raw_xml_encode, strictly_equal -from ..dataobjects import ElementData +from ..dataobjects import ElementData, DataElement, DataBindingMeta from ..converters import XMLSchemaConverter from ..xpath import XMLSchemaProxy, ElementPathMixin @@ -79,6 +79,8 @@ class XsdElement(XsdComponent, ValidationMixin, ParticleMixin, ElementPathMixin) _final = None _head_type = None + binding = None + def __init__(self, elem, schema, parent): super(XsdElement, self).__init__(elem, schema, parent) ElementPathMixin.__init__(self) @@ -377,6 +379,13 @@ def block(self): return self._block return self.schema.block_default + def create_binding(self, *bases, **attrs): + """Create data object binding for XSD element.""" + if not bases: + bases = (DataElement,) + self.binding = DataBindingMeta(self, bases, attrs) + return self.binding + def get_attribute(self, name): if name[0] != '{': return self.type.attributes[get_qname(self.type.target_namespace, name)] diff --git a/xmlschema/validators/global_maps.py b/xmlschema/validators/global_maps.py index 6f9a06dd..62393711 100644 --- a/xmlschema/validators/global_maps.py +++ b/xmlschema/validators/global_maps.py @@ -375,28 +375,31 @@ def all_errors(self): errors.extend(schema.all_errors) return errors + def create_bindings(self, *bases, **attrs): + """Creates data object bindings for the XSD elements of built schemas.""" + for xsd_element in self.iter_components(xsd_classes=XsdElement): + if xsd_element.target_namespace != XSD_NAMESPACE: + xsd_element.create_binding(*bases, **attrs) + def iter_components(self, xsd_classes=None): + """Creates an iterator for the XSD components of built schemas.""" if xsd_classes is None or isinstance(self, xsd_classes): yield self for xsd_global in self.iter_globals(): yield from xsd_global.iter_components(xsd_classes) - def iter_schemas(self): - """Creates an iterator for the schemas registered in the instance.""" - for schemas in self.namespaces.values(): - yield from schemas - def iter_globals(self): - """ - Creates an iterator for XSD global definitions/declarations. - """ + """Creates an iterator for the XSD global components of built schemas.""" for global_map in self.global_maps: yield from global_map.values() + def iter_schemas(self): + """Creates an iterator for the registered schemas.""" + for schemas in self.namespaces.values(): + yield from schemas + def register(self, schema): - """ - Registers an XMLSchema instance. - """ + """Registers an XMLSchema instance.""" try: ns_schemas = self.namespaces[schema.target_namespace] except KeyError: diff --git a/xmlschema/validators/schema.py b/xmlschema/validators/schema.py index 34a78767..18868a87 100644 --- a/xmlschema/validators/schema.py +++ b/xmlschema/validators/schema.py @@ -961,6 +961,11 @@ def get_element(self, tag, path=None, namespaces=None): else: return self.find(path, namespaces) + def create_bindings(self, *bases, **attrs): + """Creates data object bindings for XSD elements of the schema.""" + for xsd_element in self.iter_components(xsd_classes=XsdElement): + xsd_element.create_binding(*bases, **attrs) + def _parse_inclusions(self): """Processes schema document inclusions and redefinitions.""" for child in self.source.root: From af2478549e0425b861cfd524f53600ed491e5bae Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Sun, 7 Feb 2021 12:26:33 +0100 Subject: [PATCH 2/7] Optimize NamespaceView read-only mapping - Split operations by namespace value and use as_dict() as least as possible --- tests/test_namespaces.py | 4 ++++ xmlschema/namespaces.py | 34 ++++++++++++++++++++++++---------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/tests/test_namespaces.py b/tests/test_namespaces.py index 0da96f2b..f17c7239 100644 --- a/tests/test_namespaces.py +++ b/tests/test_namespaces.py @@ -211,6 +211,10 @@ def test_as_dict(self): self.assertEqual(ns_view.as_dict(), {'name1': 1, 'name2': 2}) self.assertEqual(ns_view.as_dict(True), {'{tns1}name1': 1, '{tns1}name2': 2}) + ns_view = NamespaceView(qnames, '') + self.assertEqual(ns_view.as_dict(), {'name3': 3}) + self.assertEqual(ns_view.as_dict(True), {'name3': 3}) + if __name__ == '__main__': import platform diff --git a/xmlschema/namespaces.py b/xmlschema/namespaces.py index d95942b2..d1a69211 100644 --- a/xmlschema/namespaces.py +++ b/xmlschema/namespaces.py @@ -14,7 +14,7 @@ from collections.abc import MutableMapping, Mapping from .exceptions import XMLSchemaValueError, XMLSchemaTypeError -from .helpers import get_namespace, local_name +from .helpers import local_name ### @@ -227,7 +227,8 @@ def transfer(self, namespaces): class NamespaceView(Mapping): """ - A read-only map for filtered access to a dictionary that stores objects mapped from QNames. + A read-only map for filtered access to a dictionary that stores + objects mapped from QNames in extended format. """ def __init__(self, qname_dict, namespace_uri): self.target_dict = qname_dict @@ -241,10 +242,20 @@ def __getitem__(self, key): return self.target_dict[self._key_fmt % key] def __len__(self): - return len(self.as_dict()) + if not self.namespace: + return len([k for k in self.target_dict if not k or k[0] != '{']) + return len([k for k in self.target_dict + if k and k[0] == '{' and self.namespace == k[1:k.rindex('}')]]) def __iter__(self): - return iter(self.as_dict()) + if not self.namespace: + for k in self.target_dict: + if not k or k[0] != '{': + yield k + else: + for k in self.target_dict: + if k and k[0] == '{' and self.namespace == k[1:k.rindex('}')]: + yield k[k.rindex('}') + 1:] def __repr__(self): return '%s(%s)' % (self.__class__.__name__, str(self.as_dict())) @@ -253,17 +264,20 @@ def __contains__(self, key): return self._key_fmt % key in self.target_dict def __eq__(self, other): - return self.as_dict() == dict(other.items()) + return self.as_dict() == other def as_dict(self, fqn_keys=False): - if fqn_keys: + if not self.namespace: + return { + k: v for k, v in self.target_dict.items() if not k or k[0] != '{' + } + elif fqn_keys: return { k: v for k, v in self.target_dict.items() - if self.namespace == get_namespace(k) + if k and k[0] == '{' and self.namespace == k[1:k.rindex('}')] } else: return { - k if k[0] != '{' else k[k.rindex('}') + 1:]: v - for k, v in self.target_dict.items() - if self.namespace == get_namespace(k) + k[k.rindex('}') + 1:]: v for k, v in self.target_dict.items() + if k and k[0] == '{' and self.namespace == k[1:k.rindex('}')] } From b15699742f9470f49f72f891a956e71fda5ce743 Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Sun, 7 Feb 2021 19:39:14 +0100 Subject: [PATCH 3/7] Add methods for decoding XML to data objects - Move dataelement converters to dataobjects.py module - Fix circular imports on dataobjects and validators - Refactor DataElementMeta and XsdElement.create_binding() - Add fromstring() to data binding classes - Add to_objects() to XSD validators --- CHANGELOG.rst | 2 +- tests/test_converters.py | 3 +- tests/test_dataobjects.py | 38 ++++-- xmlschema/__init__.py | 8 +- xmlschema/converters/__init__.py | 11 +- xmlschema/converters/abdera.py | 3 +- xmlschema/converters/badgerfish.py | 3 +- xmlschema/converters/columnar.py | 3 +- xmlschema/converters/dataelement.py | 104 ---------------- xmlschema/converters/default.py | 14 ++- xmlschema/converters/jsonml.py | 3 +- xmlschema/converters/parker.py | 3 +- xmlschema/converters/unordered.py | 3 +- xmlschema/dataobjects.py | 165 ++++++++++++++++++++++---- xmlschema/validators/complex_types.py | 4 +- xmlschema/validators/elements.py | 11 +- xmlschema/validators/global_maps.py | 4 + xmlschema/validators/xsdbase.py | 24 +++- 18 files changed, 233 insertions(+), 173 deletions(-) delete mode 100644 xmlschema/converters/dataelement.py diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e9c2eef8..09890e00 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ CHANGELOG ====================== * TODO: Add DataElementMeta metaclass for customizing DataElement subclasses * TODO: Implement PythonGenerator with three set of templates (xmlschema API - based, simple class hierarchy, dataclass hierarchy). + based, simple class hierarchy, dataclass hierarchy) `v1.5.0`_ (2021-02-05) ====================== diff --git a/tests/test_converters.py b/tests/test_converters.py index b610222d..f8c726c4 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -24,7 +24,8 @@ from xmlschema.converters import XMLSchemaConverter, UnorderedConverter, \ ParkerConverter, BadgerFishConverter, AbderaConverter, JsonMLConverter, \ - ColumnarConverter, DataElementConverter + ColumnarConverter +from xmlschema.dataobjects import DataElementConverter class TestConverters(unittest.TestCase): diff --git a/tests/test_dataobjects.py b/tests/test_dataobjects.py index f63291be..6720edad 100644 --- a/tests/test_dataobjects.py +++ b/tests/test_dataobjects.py @@ -19,13 +19,11 @@ from xmlschema import XMLSchema, fetch_namespaces, etree_tostring from xmlschema.helpers import is_etree_element -from xmlschema.dataobjects import DataElement, DataBindingMeta +from xmlschema.dataobjects import DataElement, DataBindingMeta, DataElementConverter from xmlschema.testing import etree_elements_assert_equal -from xmlschema.converters import DataElementConverter - -class TestDataElement(unittest.TestCase): +class TestDataObjects(unittest.TestCase): TEST_CASES_DIR = os.path.join(os.path.dirname(__file__), 'test_cases') @@ -43,6 +41,9 @@ def setUpClass(cls): def casepath(cls, relative_path): return os.path.join(cls.TEST_CASES_DIR, relative_path) + +class TestDataElement(TestDataObjects): + def test_repr(self): self.assertEqual(repr(DataElement('foo')), "DataElement(tag='foo')") @@ -216,13 +217,14 @@ def test_serialize_to_xml_source(self): self.assertTrue(xml_source.endswith('')) -class TestDataElementMeta(TestDataElement): +class TestDataBindingsMeta(TestDataObjects): def test_data_element_metaclass(self): xsd_element = self.col_schema.elements['collection'] - collection_class = DataBindingMeta(xsd_element, (DataElement,), {}) - self.assertEqual(collection_class.__name__, 'CollectionElement') - self.assertEqual(collection_class.__qualname__, 'CollectionElement') + collection_class = DataBindingMeta(xsd_element.local_name.title(), (DataElement,), + {'xsd_element': xsd_element}) + self.assertEqual(collection_class.__name__, 'Collection') + self.assertEqual(collection_class.__qualname__, 'Collection') self.assertIsNone(collection_class.__module__) self.assertEqual(collection_class.namespace, 'http://example.com/ns/collection') self.assertEqual(collection_class.xsd_version, '1.0') @@ -232,11 +234,14 @@ def test_element_binding(self): xsd_element.binding = None try: - cls = xsd_element.create_binding() + binding_class = xsd_element.create_binding() + self.assertEqual(binding_class.__name__, 'CollectionBinding') + self.assertEqual(binding_class.__qualname__, 'CollectionBinding') + self.assertIsNone(binding_class.__module__) self.assertIsNot(xsd_element.binding, DataElement) self.assertTrue(issubclass(xsd_element.binding, DataElement)) self.assertIsInstance(xsd_element.binding, DataBindingMeta) - self.assertIs(cls, xsd_element.binding) + self.assertIs(binding_class, xsd_element.binding) finally: xsd_element.binding = None @@ -244,6 +249,19 @@ def test_schema_bindings(self): schema = XMLSchema(self.col_xsd_filename) schema.maps.create_bindings() + col_element_class = schema.elements['collection'].binding + + col_data = col_element_class.fromsource(self.col_xml_filename) + + self.assertEqual(len(list(col_data.iter())), len(list(self.col_xml_root.iter()))) + for elem, data_element in zip(self.col_xml_root.iter(), col_data.iter()): + self.assertEqual(elem.tag, data_element.tag) + self.assertIsInstance(data_element, DataElement) + + self.assertIsNone( + etree_elements_assert_equal(col_data.encode(), self.col_xml_root, strict=False) + ) + if __name__ == '__main__': import platform diff --git a/xmlschema/__init__.py b/xmlschema/__init__.py index 5d69eb4d..5c0548ab 100644 --- a/xmlschema/__init__.py +++ b/xmlschema/__init__.py @@ -11,13 +11,13 @@ from . import limits from .exceptions import XMLSchemaException, XMLResourceError, XMLSchemaNamespaceError from .etree import etree_tostring -from .dataobjects import ElementData, DataElement from .resources import normalize_url, normalize_locations, fetch_resource, \ fetch_namespaces, fetch_schema_locations, fetch_schema, XMLResource from .xpath import ElementPathMixin -from .converters import XMLSchemaConverter, UnorderedConverter, \ - ParkerConverter, BadgerFishConverter, AbderaConverter, \ - JsonMLConverter, ColumnarConverter, DataElementConverter +from .converters import ElementData, XMLSchemaConverter, \ + UnorderedConverter, ParkerConverter, BadgerFishConverter, \ + AbderaConverter, JsonMLConverter, ColumnarConverter +from .dataobjects import DataElement, DataElementConverter from .documents import validate, is_valid, iter_errors, to_dict, to_json, \ from_json, XmlDocument diff --git a/xmlschema/converters/__init__.py b/xmlschema/converters/__init__.py index 39ef9659..1c7fabf1 100644 --- a/xmlschema/converters/__init__.py +++ b/xmlschema/converters/__init__.py @@ -10,16 +10,15 @@ """ This subpackage contains converter classes. """ -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter from .unordered import UnorderedConverter from .parker import ParkerConverter from .badgerfish import BadgerFishConverter from .abdera import AbderaConverter from .jsonml import JsonMLConverter from .columnar import ColumnarConverter -from .dataelement import DataElementConverter -__all__ = ['XMLSchemaConverter', 'UnorderedConverter', - 'ParkerConverter', 'BadgerFishConverter', - 'AbderaConverter', 'JsonMLConverter', - 'ColumnarConverter', 'DataElementConverter'] +__all__ = ['ElementData', 'XMLSchemaConverter', + 'UnorderedConverter', 'ParkerConverter', + 'BadgerFishConverter', 'AbderaConverter', + 'JsonMLConverter', 'ColumnarConverter'] diff --git a/xmlschema/converters/abdera.py b/xmlschema/converters/abdera.py index f01bf47f..381e7d8e 100644 --- a/xmlschema/converters/abdera.py +++ b/xmlschema/converters/abdera.py @@ -10,8 +10,7 @@ from collections.abc import MutableMapping, MutableSequence from ..exceptions import XMLSchemaValueError -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class AbderaConverter(XMLSchemaConverter): diff --git a/xmlschema/converters/badgerfish.py b/xmlschema/converters/badgerfish.py index e04047b2..715a0999 100644 --- a/xmlschema/converters/badgerfish.py +++ b/xmlschema/converters/badgerfish.py @@ -9,8 +9,7 @@ # from collections.abc import MutableMapping, MutableSequence -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class BadgerFishConverter(XMLSchemaConverter): diff --git a/xmlschema/converters/columnar.py b/xmlschema/converters/columnar.py index 1b86942b..0ac4f2a5 100644 --- a/xmlschema/converters/columnar.py +++ b/xmlschema/converters/columnar.py @@ -10,8 +10,7 @@ from collections.abc import MutableMapping, MutableSequence from ..exceptions import XMLSchemaTypeError, XMLSchemaValueError -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class ColumnarConverter(XMLSchemaConverter): diff --git a/xmlschema/converters/dataelement.py b/xmlschema/converters/dataelement.py deleted file mode 100644 index 37a53e12..00000000 --- a/xmlschema/converters/dataelement.py +++ /dev/null @@ -1,104 +0,0 @@ -# -# Copyright (c), 2016-2021, SISSA (International School for Advanced Studies). -# All rights reserved. -# This file is distributed under the terms of the MIT License. -# See the file 'LICENSE' in the root directory of the present -# distribution, or http://opensource.org/licenses/MIT. -# -# @author Davide Brunato -# -from ..exceptions import XMLSchemaValueError -from ..dataobjects import ElementData, DataElement, DataBindingMeta -from .default import XMLSchemaConverter - - -class DataElementConverter(XMLSchemaConverter): - """ - XML Schema based converter class for DataElement objects. - - :param namespaces: Map from namespace prefixes to URI. - :param data_element_class: MutableSequence subclass to use for decoded data. \ - Default is `DataElement`. - """ - data_element_class = DataElement - - def __init__(self, namespaces=None, data_element_class=None, **kwargs): - if data_element_class is not None: - self.data_element_class = data_element_class - kwargs.update(attr_prefix='', text_key='', cdata_prefix='') - super(DataElementConverter, self).__init__(namespaces, **kwargs) - - @property - def lossy(self): - return False - - @property - def losslessly(self): - return True - - def element_decode(self, data, xsd_element, xsd_type=None, level=0): - data_element = self.data_element_class( - tag=data.tag, - value=data.text, - nsmap=self.namespaces, - xsd_element=xsd_element, - xsd_type=xsd_type - ) - data_element.attrib.update((k, v) for k, v in self.map_attributes(data.attributes)) - - if (xsd_type or xsd_element.type).model_group is not None: - data_element.extend([ - value if value is not None else self.list([name]) - for name, value, _ in self.map_content(data.content) - ]) - - return data_element - - def element_encode(self, data_element, xsd_element, level=0): - self.namespaces.update(data_element.nsmap) - if not xsd_element.is_matching(data_element.tag, self._namespaces.get('')): - raise XMLSchemaValueError("Unmatched tag") - - attributes = {self.unmap_qname(k, xsd_element.attributes): v - for k, v in data_element.attrib.items()} - - data_len = len(data_element) - if not data_len: - return ElementData(data_element.tag, data_element.value, None, attributes) - - elif data_len == 1 and \ - (xsd_element.type.simple_type is not None or not - xsd_element.type.content and xsd_element.type.mixed): - return ElementData(data_element.tag, data_element.value, [], attributes) - else: - cdata_num = iter(range(1, data_len)) - content = [ - (self.unmap_qname(e.tag), e) if isinstance(e, self.data_element_class) - else (next(cdata_num), e) for e in data_element - ] - return ElementData(data_element.tag, None, content, attributes) - - -class DataBindingConverter(DataElementConverter): - """ - XML Schema based converter class for DataElementMeta metaclass objects. - - """ - def element_decode(self, data, xsd_element, xsd_type=None, level=0): - cls = xsd_element.binding or xsd_element.create_binding(self.data_element_class) - data_element = cls( - tag=data.tag, - value=data.text, - nsmap=self.namespaces, - xsd_element=xsd_element, - xsd_type=xsd_type - ) - data_element.attrib.update((k, v) for k, v in self.map_attributes(data.attributes)) - - if (xsd_type or xsd_element.type).model_group is not None: - data_element.extend([ - value if value is not None else self.list([name]) - for name, value, _ in self.map_content(data.content) - ]) - - return data_element diff --git a/xmlschema/converters/default.py b/xmlschema/converters/default.py index 31eaa0a2..1a054b02 100644 --- a/xmlschema/converters/default.py +++ b/xmlschema/converters/default.py @@ -7,13 +7,25 @@ # # @author Davide Brunato # +from collections import namedtuple from collections.abc import MutableMapping, MutableSequence from ..exceptions import XMLSchemaTypeError from ..names import XSI_NAMESPACE from ..etree import etree_element from ..namespaces import NamespaceMapper -from ..dataobjects import ElementData + + +ElementData = namedtuple('ElementData', ['tag', 'text', 'content', 'attributes']) +""" +Namedtuple for Element data interchange between decoders and converters. +The field *tag* is a string containing the Element's tag, *text* can be `None` +or a string representing the Element's text, *content* can be `None`, a list +containing the Element's children or a dictionary containing element name to +list of element contents for the Element's children (used for unordered input +data), *attributes* can be `None` or a dictionary containing the Element's +attributes. +""" class XMLSchemaConverter(NamespaceMapper): diff --git a/xmlschema/converters/jsonml.py b/xmlschema/converters/jsonml.py index 5c133bd8..d55170b5 100644 --- a/xmlschema/converters/jsonml.py +++ b/xmlschema/converters/jsonml.py @@ -10,8 +10,7 @@ from collections.abc import MutableSequence from ..exceptions import XMLSchemaValueError -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class JsonMLConverter(XMLSchemaConverter): diff --git a/xmlschema/converters/parker.py b/xmlschema/converters/parker.py index 2810f06c..d799adda 100644 --- a/xmlschema/converters/parker.py +++ b/xmlschema/converters/parker.py @@ -9,8 +9,7 @@ # from collections.abc import MutableMapping, MutableSequence -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class ParkerConverter(XMLSchemaConverter): diff --git a/xmlschema/converters/unordered.py b/xmlschema/converters/unordered.py index 546d1f2e..717ca2fd 100644 --- a/xmlschema/converters/unordered.py +++ b/xmlschema/converters/unordered.py @@ -9,8 +9,7 @@ # from collections.abc import MutableMapping, MutableSequence -from ..dataobjects import ElementData -from .default import XMLSchemaConverter +from .default import ElementData, XMLSchemaConverter class UnorderedConverter(XMLSchemaConverter): diff --git a/xmlschema/dataobjects.py b/xmlschema/dataobjects.py index a21ba5a8..47e3e332 100644 --- a/xmlschema/dataobjects.py +++ b/xmlschema/dataobjects.py @@ -8,25 +8,14 @@ # @author Davide Brunato # from abc import ABCMeta -from collections import namedtuple from collections.abc import MutableSequence from elementpath import XPathContext, XPath2Parser -from .exceptions import XMLSchemaValueError +from .exceptions import XMLSchemaAttributeError, XMLSchemaTypeError, XMLSchemaValueError from .etree import etree_tostring from .helpers import get_namespace, get_prefixed_qname, local_name, raw_xml_encode - - -ElementData = namedtuple('ElementData', ['tag', 'text', 'content', 'attributes']) -""" -Namedtuple for Element data interchange between decoders and converters. -The field *tag* is a string containing the Element's tag, *text* can be `None` -or a string representing the Element's text, *content* can be `None`, a list -containing the Element's children or a dictionary containing element name to -list of element contents for the Element's children (used for unordered input -data), *attributes* can be `None` or a dictionary containing the Element's -attributes. -""" +from .converters import ElementData, XMLSchemaConverter +from . import validators class DataElement(MutableSequence): @@ -35,6 +24,8 @@ class DataElement(MutableSequence): """ value = None tail = None + xsd_element = None + _xsd_type = None def __init__(self, tag, value=None, attrib=None, nsmap=None, xsd_element=None, xsd_type=None): super(DataElement, self).__init__() @@ -50,8 +41,23 @@ def __init__(self, tag, value=None, attrib=None, nsmap=None, xsd_element=None, x if nsmap: self.nsmap.update(nsmap) - self.xsd_element = xsd_element - self._xsd_type = xsd_type + if xsd_element is None: + pass + elif not isinstance(xsd_element, validators.XsdElement): + msg = "argument 'xsd_element' must be an {!r} instance" + raise XMLSchemaTypeError(msg.format(validators.XsdElement)) + elif self.xsd_element is None: + self.xsd_element = xsd_element + elif xsd_element is not self.xsd_element: + raise XMLSchemaValueError("the class has a binding with a different XSD element") + + if xsd_type is None: + pass + elif not isinstance(xsd_type, validators.XsdType): + msg = "argument 'xsd_type' must be an {!r} instance" + raise XMLSchemaTypeError(msg.format(validators.XsdType)) + else: + self._xsd_type = xsd_type def __getitem__(self, i): return self._children[i] @@ -85,6 +91,10 @@ def get(self, key, default=None): """Gets a data element attribute.""" return self.attrib.get(key, default) + @property + def xsd_version(self): + return '1.0' if self.xsd_element is None else self.xsd_element.xsd_version + @property def namespace(self): """The element's namespace.""" @@ -119,6 +129,9 @@ def xsd_type(self, xsd_type): self._xsd_type = xsd_type def encode(self, **kwargs): + if 'converter' not in kwargs: + kwargs['converter'] = DataElementConverter + if self._xsd_type is None: if self.xsd_element is not None: return self.xsd_element.encode(self, **kwargs) @@ -210,13 +223,119 @@ def iterchildren(self, tag=None): class DataBindingMeta(ABCMeta): """Metaclass for creating classes with bindings to XSD elements.""" - def __new__(mcs, xsd_element, bases, attrs): - name = xsd_element.local_name - class_name = '{}Element'.format(name.replace('_', '').title()) + + def __new__(mcs, name, bases, attrs): + try: + xsd_element = attrs['xsd_element'] + except KeyError: + msg = "attribute 'xsd_element' is required for an XSD data binding class" + raise XMLSchemaAttributeError(msg) from None + + if not isinstance(xsd_element, validators.XsdElement): + raise XMLSchemaTypeError("{!r} is not an XSD element".format(xsd_element)) attrs['__module__'] = None - attrs['namespace'] = xsd_element.target_namespace - attrs['xsd_version'] = xsd_element.xsd_version - cls = super(DataBindingMeta, mcs).__new__(mcs, class_name, bases, attrs) - return cls + return super(DataBindingMeta, mcs).__new__(mcs, name, bases, attrs) + + def __init__(cls, name, bases, attrs): + super(DataBindingMeta, cls).__init__(name, bases, attrs) + cls.xsd_version = cls.xsd_element.xsd_version + cls.namespace = cls.xsd_element.target_namespace + def fromsource(cls, source, **kwargs): + if 'converter' not in kwargs: + kwargs['converter'] = DataBindingConverter + return cls.xsd_element.schema.decode(source, **kwargs) + + +class DataElementConverter(XMLSchemaConverter): + """ + XML Schema based converter class for DataElement objects. + + :param namespaces: a dictionary map from namespace prefixes to URI. + :param data_element_class: MutableSequence subclass to use for decoded data. \ + Default is `DataElement`. + """ + data_element_class = DataElement + + def __init__(self, namespaces=None, data_element_class=None, **kwargs): + if data_element_class is not None: + self.data_element_class = data_element_class + kwargs.update(attr_prefix='', text_key='', cdata_prefix='') + super(DataElementConverter, self).__init__(namespaces, **kwargs) + + @property + def lossy(self): + return False + + @property + def losslessly(self): + return True + + def element_decode(self, data, xsd_element, xsd_type=None, level=0): + data_element = self.data_element_class( + tag=data.tag, + value=data.text, + nsmap=self.namespaces, + xsd_element=xsd_element, + xsd_type=xsd_type + ) + data_element.attrib.update((k, v) for k, v in self.map_attributes(data.attributes)) + + if (xsd_type or xsd_element.type).model_group is not None: + data_element.extend([ + value if value is not None else self.list([name]) + for name, value, _ in self.map_content(data.content) + ]) + + return data_element + + def element_encode(self, data_element, xsd_element, level=0): + self.namespaces.update(data_element.nsmap) + if not xsd_element.is_matching(data_element.tag, self._namespaces.get('')): + raise XMLSchemaValueError("Unmatched tag") + + attributes = {self.unmap_qname(k, xsd_element.attributes): v + for k, v in data_element.attrib.items()} + + data_len = len(data_element) + if not data_len: + return ElementData(data_element.tag, data_element.value, None, attributes) + + elif data_len == 1 and \ + (xsd_element.type.simple_type is not None or not + xsd_element.type.content and xsd_element.type.mixed): + return ElementData(data_element.tag, data_element.value, [], attributes) + else: + cdata_num = iter(range(1, data_len)) + content = [ + (self.unmap_qname(e.tag), e) if isinstance(e, self.data_element_class) + else (next(cdata_num), e) for e in data_element + ] + return ElementData(data_element.tag, None, content, attributes) + + +class DataBindingConverter(DataElementConverter): + """ + A :class:`DataElementConverter` that uses XML data binding classes for + decoding. Takes the same arguments of its parent class but the argument + *data_element_class* is used for define the base for creating the missing + XML binding classes. + """ + def element_decode(self, data, xsd_element, xsd_type=None, level=0): + cls = xsd_element.binding or xsd_element.create_binding(self.data_element_class) + data_element = cls( + tag=data.tag, + value=data.text, + nsmap=self.namespaces, + xsd_type=xsd_type + ) + data_element.attrib.update((k, v) for k, v in self.map_attributes(data.attributes)) + + if (xsd_type or xsd_element.type).model_group is not None: + data_element.extend([ + value if value is not None else self.list([name]) + for name, value, _ in self.map_content(data.content) + ]) + + return data_element diff --git a/xmlschema/validators/complex_types.py b/xmlschema/validators/complex_types.py index f13929fb..76e9d522 100644 --- a/xmlschema/validators/complex_types.py +++ b/xmlschema/validators/complex_types.py @@ -15,7 +15,7 @@ XSD_RESTRICTION, XSD_COMPLEX_TYPE, XSD_EXTENSION, XSD_ANY_TYPE, XSD_ASSERT, \ XSD_UNTYPED_ATOMIC, XSD_SIMPLE_CONTENT, XSD_OPEN_CONTENT, XSD_ANNOTATION from ..helpers import get_prefixed_qname, get_qname, local_name -from ..dataobjects import DataElement +from .. import dataobjects from .exceptions import XMLSchemaDecodeError from .helpers import get_xsd_derivation_attribute @@ -680,7 +680,7 @@ def iter_encode(self, obj, validation='lax', **kwargs): xsd_element = self.schema.create_element(name, parent=self, form='unqualified') xsd_element.type = self - if isinstance(value, MutableSequence) and not isinstance(value, DataElement): + if isinstance(value, MutableSequence) and not isinstance(value, dataobjects.DataElement): try: results = [x for item in value for x in xsd_element.iter_encode( item, validation, **kwargs diff --git a/xmlschema/validators/elements.py b/xmlschema/validators/elements.py index 464ab84e..55ba8116 100644 --- a/xmlschema/validators/elements.py +++ b/xmlschema/validators/elements.py @@ -23,8 +23,8 @@ from ..etree import etree_element from ..helpers import get_qname, get_namespace, etree_iter_location_hints, \ raw_xml_encode, strictly_equal -from ..dataobjects import ElementData, DataElement, DataBindingMeta -from ..converters import XMLSchemaConverter +from .. import dataobjects +from ..converters import ElementData, XMLSchemaConverter from ..xpath import XMLSchemaProxy, ElementPathMixin from .exceptions import XMLSchemaValidationError, XMLSchemaTypeTableWarning @@ -382,8 +382,11 @@ def block(self): def create_binding(self, *bases, **attrs): """Create data object binding for XSD element.""" if not bases: - bases = (DataElement,) - self.binding = DataBindingMeta(self, bases, attrs) + bases = (dataobjects.DataElement,) + attrs['xsd_element'] = self + class_name = '{}Binding'.format(self.local_name.title().replace('_', '')) + + self.binding = dataobjects.DataBindingMeta(class_name, bases, attrs) return self.binding def get_attribute(self, name): diff --git a/xmlschema/validators/global_maps.py b/xmlschema/validators/global_maps.py index 62393711..d5fc1dd1 100644 --- a/xmlschema/validators/global_maps.py +++ b/xmlschema/validators/global_maps.py @@ -381,6 +381,10 @@ def create_bindings(self, *bases, **attrs): if xsd_element.target_namespace != XSD_NAMESPACE: xsd_element.create_binding(*bases, **attrs) + def clear_bindings(self): + for xsd_element in self.iter_components(xsd_classes=XsdElement): + xsd_element.binding = None + def iter_components(self, xsd_classes=None): """Creates an iterator for the XSD components of built schemas.""" if xsd_classes is None or isinstance(self, xsd_classes): diff --git a/xmlschema/validators/xsdbase.py b/xmlschema/validators/xsdbase.py index 1a105dcb..6a49eb26 100644 --- a/xmlschema/validators/xsdbase.py +++ b/xmlschema/validators/xsdbase.py @@ -19,8 +19,7 @@ XSD_OVERRIDE, XSD_NOTATION_TYPE, XSD_DECIMAL from ..etree import is_etree_element, etree_tostring from ..helpers import get_qname, local_name, get_prefixed_qname -from ..dataobjects import DataElement -from ..converters import DataElementConverter +from .. import dataobjects from .exceptions import XMLSchemaParseError, XMLSchemaValidationError XSD_TYPE_DERIVATIONS = {'extension', 'restriction'} @@ -838,6 +837,24 @@ def decode(self, source, validation='strict', **kwargs): return (result, errors) if validation == 'lax' else result + def to_objects(self, source, with_bindings=False, **kwargs): + """ + Decodes XML data to Python data objects. + + :param source: the XML data. Can be a string for an attribute or for a simple \ + type components or a dictionary for an attribute group or an ElementTree's \ + Element for other components. + :param with_bindings: if `True` is provided the decoding is done using \ + :class:`DataBindingConverter` that used XML data binding classes. For \ + default the objects are intances of :class:`DataElement` and uses the \ + :class:`DataElementConverter`. + :param kwargs: other optional keyword arguments for the method \ + :func:`iter_decode`, except the argument *converter*. + """ + if with_bindings: + return self.decode(source, converter=dataobjects.DataBindingConverter, **kwargs) + return self.decode(source, converter=dataobjects.DataElementConverter, **kwargs) + def encode(self, obj, validation='strict', **kwargs): """ Encodes data to XML. @@ -853,9 +870,6 @@ def encode(self, obj, validation='strict', **kwargs): component, or also if it's invalid when ``validation='strict'`` is provided. """ check_validation_mode(validation) - if 'converter' not in kwargs and isinstance(obj, DataElement): - kwargs['converter'] = DataElementConverter - result, errors = None, [] for result in self.iter_encode(obj, validation=validation, **kwargs): # pragma: no cover if not isinstance(result, XMLSchemaValidationError): From 7feb3454c7031e23b8ab06fb13ef295bd3a669f4 Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Wed, 10 Feb 2021 13:27:54 +0100 Subject: [PATCH 4/7] Add validation API to DataElement - Limit modifications of XSD schema bindings xsd_element and xsd_type - Validation use the encode API instead of decode - Fix invalid value type with 'lax' validation mode, keeping result string if it's equivalent. --- tests/test_converters.py | 6 +- tests/test_dataobjects.py | 121 +++++++++++++-------- tests/validation/test_encoding.py | 22 ++-- xmlschema/dataobjects.py | 152 +++++++++++++++++++-------- xmlschema/validators/simple_types.py | 34 +++--- xmlschema/validators/xsdbase.py | 6 +- 6 files changed, 220 insertions(+), 121 deletions(-) diff --git a/tests/test_converters.py b/tests/test_converters.py index f8c726c4..94f402fe 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -9,8 +9,8 @@ # @author Davide Brunato # import unittest -import os import xml.etree.ElementTree as ElementTree +from pathlib import Path try: import lxml.etree as lxml_etree @@ -30,8 +30,6 @@ class TestConverters(unittest.TestCase): - TEST_CASES_DIR = os.path.join(os.path.dirname(__file__), 'test_cases') - @classmethod def setUpClass(cls): cls.col_xsd_filename = cls.casepath('examples/collection/collection.xsd') @@ -43,7 +41,7 @@ def setUpClass(cls): @classmethod def casepath(cls, relative_path): - return os.path.join(cls.TEST_CASES_DIR, relative_path) + return str(Path(__file__).parent.joinpath('test_cases', relative_path)) def test_element_class_argument(self): converter = XMLSchemaConverter() diff --git a/tests/test_dataobjects.py b/tests/test_dataobjects.py index 6720edad..c1e1beec 100644 --- a/tests/test_dataobjects.py +++ b/tests/test_dataobjects.py @@ -9,40 +9,23 @@ # @author Davide Brunato # import unittest -import os import xml.etree.ElementTree as ElementTree +from pathlib import Path try: import lxml.etree as lxml_etree except ImportError: lxml_etree = None -from xmlschema import XMLSchema, fetch_namespaces, etree_tostring +from xmlschema import XMLSchema, fetch_namespaces, etree_tostring, \ + XMLSchemaValidationError, DataElement, DataElementConverter + from xmlschema.helpers import is_etree_element -from xmlschema.dataobjects import DataElement, DataBindingMeta, DataElementConverter +from xmlschema.dataobjects import DataBindingMeta, DataBindingConverter from xmlschema.testing import etree_elements_assert_equal -class TestDataObjects(unittest.TestCase): - - TEST_CASES_DIR = os.path.join(os.path.dirname(__file__), 'test_cases') - - @classmethod - def setUpClass(cls): - cls.col_xsd_filename = cls.casepath('examples/collection/collection.xsd') - cls.col_xml_filename = cls.casepath('examples/collection/collection.xml') - cls.col_xml_root = ElementTree.parse(cls.col_xml_filename).getroot() - cls.col_lxml_root = lxml_etree.parse(cls.col_xml_filename).getroot() - cls.col_nsmap = fetch_namespaces(cls.col_xml_filename) - cls.col_namespace = cls.col_nsmap['col'] - cls.col_schema = XMLSchema(cls.col_xsd_filename, converter=DataElementConverter) - - @classmethod - def casepath(cls, relative_path): - return os.path.join(cls.TEST_CASES_DIR, relative_path) - - -class TestDataElement(TestDataObjects): +class TestDataElementInterface(unittest.TestCase): def test_repr(self): self.assertEqual(repr(DataElement('foo')), "DataElement(tag='foo')") @@ -71,6 +54,25 @@ def test_text_value(self): self.assertEqual(DataElement('foo', value=False).text, 'false') self.assertEqual(DataElement('foo', value=10.0).text, '10.0') + +class TestDataObjects(unittest.TestCase): + + converter = DataElementConverter + + @classmethod + def setUpClass(cls): + cls.col_xsd_filename = cls.casepath('examples/collection/collection.xsd') + cls.col_xml_filename = cls.casepath('examples/collection/collection.xml') + cls.col_xml_root = ElementTree.parse(cls.col_xml_filename).getroot() + cls.col_lxml_root = lxml_etree.parse(cls.col_xml_filename).getroot() + cls.col_nsmap = fetch_namespaces(cls.col_xml_filename) + cls.col_namespace = cls.col_nsmap['col'] + cls.col_schema = XMLSchema(cls.col_xsd_filename, converter=cls.converter) + + @classmethod + def casepath(cls, relative_path): + return str(Path(__file__).parent.joinpath('test_cases', relative_path)) + def test_namespace(self): col_data = self.col_schema.decode(self.col_xml_filename) @@ -153,14 +155,10 @@ def test_schema_bindings(self): col_data = self.col_schema.decode(self.col_xml_filename) self.assertIs(col_data.xsd_type, self.col_schema.elements['collection'].type) - self.assertIs(col_data._xsd_type, self.col_schema.elements['collection'].type) - - col_data.xsd_type = None - self.assertIs(col_data.xsd_type, self.col_schema.elements['collection'].type) - self.assertIsNone(col_data._xsd_type) - col_data.xsd_element = None - self.assertIsNone(col_data.xsd_type) + with self.assertRaises(ValueError) as ec: + col_data.xsd_type = col_data[0].xsd_type + self.assertEqual(str(ec.exception), "the instance is already bound to another XSD type") def test_encode_to_element_tree(self): col_data = self.col_schema.decode(self.col_xml_filename) @@ -170,18 +168,13 @@ def test_encode_to_element_tree(self): self.assertIsInstance(etree_tostring(obj), str) self.assertIsNone(etree_elements_assert_equal(obj, self.col_xml_root, strict=False)) - col_data.xsd_type = None - obj = col_data.encode() - self.assertTrue(is_etree_element(obj)) - self.assertIsInstance(etree_tostring(obj), str) - self.assertIsNone(etree_elements_assert_equal(obj, self.col_xml_root, strict=False)) + with self.assertRaises(ValueError) as ec: + col_data.xsd_type = col_data[0].xsd_type + self.assertEqual(str(ec.exception), "the instance is already bound to another XSD type") - col_data.xsd_type = col_data.xsd_element.type - col_data.xsd_element = None - obj = col_data.encode() - self.assertTrue(is_etree_element(obj)) - self.assertIsInstance(etree_tostring(obj), str) - self.assertIsNone(etree_elements_assert_equal(obj, self.col_xml_root, strict=False)) + with self.assertRaises(ValueError) as ec: + col_data.xsd_element = col_data[0].xsd_element + self.assertEqual(str(ec.exception), "the instance is already bound to another XSD element") any_data = DataElement('a') any_data.append(DataElement('b1', 1999)) @@ -190,7 +183,7 @@ def test_encode_to_element_tree(self): with self.assertRaises(ValueError) as ec: any_data.encode() - self.assertIn("no schema bindings and valition mode is not 'skip'", str(ec.exception)) + self.assertIn("has no schema bindings", str(ec.exception)) root = ElementTree.XML('1999alphatrue') @@ -216,8 +209,50 @@ def test_serialize_to_xml_source(self): self.assertTrue(xml_source.startswith('')) + def test_validation(self): + col_data = self.col_schema.decode(self.col_xml_filename) + + self.assertIsNone(col_data.validate()) + self.assertTrue(col_data.is_valid()) + self.assertListEqual(list(col_data.iter_errors()), []) + + def test_invalid_value_type(self): + col_data = self.col_schema.decode(self.col_xml_filename) + self.assertTrue(col_data.is_valid()) + + col_data[0][0].value = '1' + with self.assertRaises(XMLSchemaValidationError) as ec: + col_data.validate() + self.assertIn("'1' is not an instance of ", str(ec.exception)) + self.assertFalse(col_data.is_valid()) + + errors = list(col_data.iter_errors()) + self.assertEqual(len(errors), 1) + self.assertIn("'1' is not an instance of ", str(errors[0])) + + col_data.find('object/position').value = 1 + self.assertTrue(col_data.is_valid()) + + def test_missing_child(self): + col_data = self.col_schema.decode(self.col_xml_filename) + self.assertTrue(col_data.is_valid()) + + title = col_data[0].pop(1) + with self.assertRaises(XMLSchemaValidationError) as ec: + col_data.validate() + self.assertIn("Unexpected child with tag 'year' at position 2", str(ec.exception)) + self.assertFalse(col_data.is_valid()) + errors = list(col_data.iter_errors()) + self.assertEqual(len(errors), 1) + self.assertIn("Unexpected child with tag 'year' at position 2", str(errors[0])) + + col_data[0].insert(1, title) + self.assertTrue(col_data.is_valid()) + + +class TestDataBindings(TestDataObjects): -class TestDataBindingsMeta(TestDataObjects): + converter = DataBindingConverter def test_data_element_metaclass(self): xsd_element = self.col_schema.elements['collection'] diff --git a/tests/validation/test_encoding.py b/tests/validation/test_encoding.py index 0c0a8e70..b740d7e5 100644 --- a/tests/validation/test_encoding.py +++ b/tests/validation/test_encoding.py @@ -177,26 +177,26 @@ def test_base64_binary_type(self): def test_list_types(self): list_of_strings = self.st_schema.types['list_of_strings'] - self.check_encode(list_of_strings, (10, 25, 40), u'', validation='lax') - self.check_encode(list_of_strings, (10, 25, 40), u'10 25 40', validation='skip') - self.check_encode(list_of_strings, ['a', 'b', 'c'], u'a b c', validation='skip') + self.check_encode(list_of_strings, (10, 25, 40), '10 25 40', validation='lax') + self.check_encode(list_of_strings, (10, 25, 40), '10 25 40', validation='skip') + self.check_encode(list_of_strings, ['a', 'b', 'c'], 'a b c', validation='skip') list_of_integers = self.st_schema.types['list_of_integers'] - self.check_encode(list_of_integers, (10, 25, 40), u'10 25 40') + self.check_encode(list_of_integers, (10, 25, 40), '10 25 40') self.check_encode(list_of_integers, (10, 25.0, 40), XMLSchemaValidationError) - self.check_encode(list_of_integers, (10, 25.0, 40), u'10 25 40', validation='lax') + self.check_encode(list_of_integers, (10, 25.0, 40), '10 25 40', validation='lax') list_of_floats = self.st_schema.types['list_of_floats'] - self.check_encode(list_of_floats, [10.1, 25.0, 40.0], u'10.1 25.0 40.0') - self.check_encode(list_of_floats, [10.1, 25, 40.0], u'10.1 25.0 40.0', validation='lax') - self.check_encode(list_of_floats, [10.1, False, 40.0], u'10.1 0.0 40.0', validation='lax') + self.check_encode(list_of_floats, [10.1, 25.0, 40.0], '10.1 25.0 40.0') + self.check_encode(list_of_floats, [10.1, 25, 40.0], '10.1 25.0 40.0', validation='lax') + self.check_encode(list_of_floats, [10.1, False, 40.0], '10.1 0.0 40.0', validation='lax') list_of_booleans = self.st_schema.types['list_of_booleans'] - self.check_encode(list_of_booleans, [True, False, True], u'true false true') + self.check_encode(list_of_booleans, [True, False, True], 'true false true') self.check_encode(list_of_booleans, [10, False, True], XMLSchemaEncodeError) - self.check_encode(list_of_booleans, [True, False, 40.0], u'true false', validation='lax') + self.check_encode(list_of_booleans, [True, False, 40.0], 'true false', validation='lax') self.check_encode(list_of_booleans, [True, False, 40.0], - u'true false 40.0', validation='skip') + 'true false 40.0', validation='skip') def test_union_types(self): integer_or_float = self.st_schema.types['integer_or_float'] diff --git a/xmlschema/dataobjects.py b/xmlschema/dataobjects.py index 47e3e332..e838f7be 100644 --- a/xmlschema/dataobjects.py +++ b/xmlschema/dataobjects.py @@ -21,11 +21,20 @@ class DataElement(MutableSequence): """ Data Element, an Element like object with decoded data and schema bindings. + + :param tag: a string containing a QName in extended format. + :param value: the simple typed value of the element. + :param attrib: the typed attributes of the element. + :param nsmap: an optional map from prefixes to namespaces. + :param xsd_element: an optional XSD element association. + :param xsd_type: an optional XSD type association. Can be provided \ + also if the instance is not bound with an XSD element. """ value = None tail = None xsd_element = None - _xsd_type = None + xsd_type = None + _encoder = None def __init__(self, tag, value=None, attrib=None, nsmap=None, xsd_element=None, xsd_type=None): super(DataElement, self).__init__() @@ -41,23 +50,11 @@ def __init__(self, tag, value=None, attrib=None, nsmap=None, xsd_element=None, x if nsmap: self.nsmap.update(nsmap) - if xsd_element is None: - pass - elif not isinstance(xsd_element, validators.XsdElement): - msg = "argument 'xsd_element' must be an {!r} instance" - raise XMLSchemaTypeError(msg.format(validators.XsdElement)) - elif self.xsd_element is None: + if xsd_element is not None: self.xsd_element = xsd_element - elif xsd_element is not self.xsd_element: - raise XMLSchemaValueError("the class has a binding with a different XSD element") - - if xsd_type is None: - pass - elif not isinstance(xsd_type, validators.XsdType): - msg = "argument 'xsd_type' must be an {!r} instance" - raise XMLSchemaTypeError(msg.format(validators.XsdType)) - else: - self._xsd_type = xsd_type + self.xsd_type = xsd_type or xsd_element.type + elif xsd_type is not None: + self.xsd_type = xsd_type def __getitem__(self, i): return self._children[i] @@ -82,6 +79,32 @@ def __repr__(self): def __iter__(self): yield from self._children + def __setattr__(self, key, value): + if key == 'xsd_element': + if not isinstance(value, validators.XsdElement): + msg = "attribute 'xsd_element' must be an {!r} instance" + raise XMLSchemaTypeError(msg.format(validators.XsdElement)) + elif self.xsd_element is not None and self.xsd_element is not value: + raise XMLSchemaValueError("the instance is already bound to another XSD element") + elif self.xsd_type is not None and self.xsd_type is not value.type: + raise XMLSchemaValueError("the instance is already bound to another XSD type") + + elif key == 'xsd_type': + if not isinstance(value, validators.XsdType): + msg = "attribute 'xsd_type' must be an {!r} instance" + raise XMLSchemaTypeError(msg.format(validators.XsdType)) + elif self.xsd_type is not None and self.xsd_type is not value: + raise XMLSchemaValueError("the instance is already bound to another XSD type") + elif self.xsd_element is None or value is not self.xsd_element.type: + self._encoder = value.schema.create_element( + self.tag, parent=value, form='unqualified' + ) + self._encoder.type = value + else: + self._encoder = self.xsd_element + + super(DataElement, self).__setattr__(key, value) + @property def text(self): """The string value of the data element.""" @@ -91,6 +114,10 @@ def get(self, key, default=None): """Gets a data element attribute.""" return self.attrib.get(key, default) + def set(self, key, value): + """Sets a data element attribute.""" + self.attrib[key] = value + @property def xsd_version(self): return '1.0' if self.xsd_element is None else self.xsd_element.xsd_version @@ -117,40 +144,75 @@ def local_name(self): """The local part of the tag.""" return local_name(self.tag) - @property - def xsd_type(self): - if self._xsd_type is not None: - return self._xsd_type - elif self.xsd_element is not None: - return self.xsd_element.type + def validate(self, use_defaults=True, namespaces=None): + """ + Validates the XML data object. + + :param use_defaults: whether to use default values for filling missing data. + :param namespaces: is an optional mapping from namespace prefix to URI. + :raises: :exc:`XMLSchemaValidationError` if XML data object is not valid. + :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. + """ + for error in self.iter_errors(use_defaults, namespaces): + raise error - @xsd_type.setter - def xsd_type(self, xsd_type): - self._xsd_type = xsd_type + def is_valid(self, use_defaults=True, namespaces=None): + """ + Like :meth:`validate` except it does not raise an exception on validation + error but returns ``True`` if the XML data object is valid, ``False`` if + it's invalid. - def encode(self, **kwargs): + :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. + """ + return next(self.iter_errors(use_defaults, namespaces), None) is None + + def iter_errors(self, use_defaults=True, namespaces=None): + """ + Generates a sequence of validation errors if the XML data object is invalid. + + :param use_defaults: whether to use default values for filling missing data. + :param namespaces: is an optional mapping from namespace prefix to URI. + :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. + """ + if self._encoder is None: + raise XMLSchemaValueError("{!r} has no schema bindings".format(self)) + + kwargs = {'converter': DataElementConverter} + if not use_defaults: + kwargs['use_defaults'] = False + if namespaces: + kwargs['namespaces'] = namespaces + + for result in self._encoder.iter_encode(self, **kwargs): + if isinstance(result, validators.XMLSchemaValidationError): + yield result + else: + del result + + def encode(self, validation='strict', **kwargs): + """ + Encodes the data object to XML. + + :param validation: the validation mode. Can be 'lax', 'strict' or 'skip. + :param kwargs: optional keyword arguments for the method :func:`iter_encode` \ + of :class:`XsdElement`. + :return: An ElementTree's Element. If *validation* argument is 'lax' a \ + 2-items tuple is returned, where the first item is the encoded object and \ + the second item is a list with validation errors. + :raises: :exc:`XMLSchemaValidationError` if the object is invalid \ + and ``validation='strict'``. + """ if 'converter' not in kwargs: kwargs['converter'] = DataElementConverter - if self._xsd_type is None: - if self.xsd_element is not None: - return self.xsd_element.encode(self, **kwargs) - elif self.xsd_element is not None and self.xsd_element.type is self._xsd_type: - return self.xsd_element.encode(self, **kwargs) + if self._encoder is not None: + encoder = self._encoder + elif validation == 'skip': + encoder = validators.XMLSchema.builtin_types()['anyType'] else: - xsd_element = self._xsd_type.schema.create_element( - self.tag, parent=self._xsd_type, form='unqualified' - ) - xsd_element.type = self._xsd_type - return xsd_element.encode(self, **kwargs) - - if kwargs.get('validation') != 'skip': - msg = "{!r} has no schema bindings and valition mode is not 'skip'" - raise XMLSchemaValueError(msg.format(self)) - - from . import XMLSchema - any_type = XMLSchema.builtin_types()['anyType'] - return any_type.encode(self, **kwargs) + raise XMLSchemaValueError("{!r} has no schema bindings".format(self)) + + return encoder.encode(self, validation=validation, **kwargs) to_etree = encode diff --git a/xmlschema/validators/simple_types.py b/xmlschema/validators/simple_types.py index 8655884a..d1c5dfd3 100644 --- a/xmlschema/validators/simple_types.py +++ b/xmlschema/validators/simple_types.py @@ -663,19 +663,19 @@ def iter_decode(self, obj, validation='lax', **kwargs): if not id_map[obj]: id_map[obj] = 1 else: - reason = "Duplicated xs:ID value {!r}".format(obj) + reason = "duplicated xs:ID value {!r}".format(obj) yield self.validation_error(validation, error=reason, obj=obj) else: if not id_map[obj]: id_map[obj] = 1 id_list.append(obj) if len(id_list) > 1 and self.xsd_version == '1.0': - reason = "No more than one attribute of type ID should " \ + reason = "no more than one attribute of type ID should " \ "be present in an element" yield self.validation_error(validation, reason, obj, **kwargs) elif obj not in id_list or self.xsd_version == '1.0': - reason = "Duplicated xs:ID value {!r}".format(obj) + reason = "duplicated xs:ID value {!r}".format(obj) yield self.validation_error(validation, error=reason, obj=obj) yield result @@ -694,28 +694,32 @@ def iter_encode(self, obj, validation='lax', **kwargs): elif isinstance(obj, bool): types_ = self.instance_types if types_ is not bool or (isinstance(types_, tuple) and bool in types_): - reason = "boolean value {!r} requires a {!r} decoder.".format(obj, bool) + reason = "boolean value {!r} requires a {!r} decoder".format(obj, bool) yield XMLSchemaEncodeError(self, obj, self.from_python, reason) obj = self.python_type(obj) elif not isinstance(obj, self.instance_types): - reason = "{!r} is not an instance of {!r}.".format(obj, self.instance_types) + reason = "{!r} is not an instance of {!r}".format(obj, self.instance_types) yield XMLSchemaEncodeError(self, obj, self.from_python, reason) + try: value = self.python_type(obj) - if value != obj: + if value != obj and not isinstance(value, str) \ + and not isinstance(obj, (str, bytes)): raise ValueError() - else: - obj = value - except ValueError: - yield XMLSchemaEncodeError(self, obj, self.from_python) - yield None - return - except TypeError: - reason = "Invalid value {!r}".format(obj) - yield self.validation_error(validation, error=reason, obj=obj) + obj = value + except (ValueError, TypeError) as err: + yield XMLSchemaEncodeError(self, obj, self.from_python, reason=str(err)) yield None return + else: + if value == obj or str(value) == str(obj): + obj = value + else: + reason = "Invalid value {!r}".format(obj) + yield XMLSchemaEncodeError(self, obj, self.from_python, reason) + yield None + return for validator in self.validators: try: diff --git a/xmlschema/validators/xsdbase.py b/xmlschema/validators/xsdbase.py index 6a49eb26..f4302395 100644 --- a/xmlschema/validators/xsdbase.py +++ b/xmlschema/validators/xsdbase.py @@ -768,14 +768,14 @@ def validate(self, source, use_defaults=True, namespaces=None): for an attribute or a simple type validators, or an ElementTree's Element otherwise. :param use_defaults: indicates whether to use default values for filling missing data. :param namespaces: is an optional mapping from namespace prefix to URI. - :raises: :exc:`XMLSchemaValidationError` if XML *data* instance is not a valid. + :raises: :exc:`XMLSchemaValidationError` if XML *data* instance is not valid. """ for error in self.iter_errors(source, use_defaults=use_defaults, namespaces=namespaces): raise error def is_valid(self, source, use_defaults=True, namespaces=None): """ - Like :meth:`validate` except that do not raises an exception but returns ``True`` if + Like :meth:`validate` except that does not raise an exception but returns ``True`` if the XML document is valid, ``False`` if it's invalid. :param source: the source of XML data. For a schema can be a path \ @@ -895,7 +895,7 @@ def iter_decode(self, source, validation='lax', **kwargs): def iter_encode(self, obj, validation='lax', **kwargs): """ - Creates an iterator for Encode data to an Element. + Creates an iterator for encoding data to an Element tree. :param obj: The data that has to be encoded. :param validation: The validation mode. Can be 'lax', 'strict' or 'skip'. From 581f76410a69f8688d4bf6d004bb354c7e6091da Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Wed, 10 Feb 2021 22:28:06 +0100 Subject: [PATCH 5/7] Add max_depth=None for DataElement validation - Added max_depth to iter_encode() of XsdElement and XsdGroup --- tests/test_dataobjects.py | 17 +++++++++++++++++ xmlschema/dataobjects.py | 16 +++++++++++----- xmlschema/validators/elements.py | 17 +++++++++++++---- xmlschema/validators/groups.py | 4 ++++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/tests/test_dataobjects.py b/tests/test_dataobjects.py index c1e1beec..1bc9cb00 100644 --- a/tests/test_dataobjects.py +++ b/tests/test_dataobjects.py @@ -249,6 +249,23 @@ def test_missing_child(self): col_data[0].insert(1, title) self.assertTrue(col_data.is_valid()) + def test_max_depth_validation(self): + col_data = self.col_schema.decode(self.col_xml_filename) + self.assertTrue(col_data.is_valid()) + + for child in col_data: + child.clear() + + self.assertFalse(col_data.is_valid()) + self.assertTrue(col_data.is_valid(max_depth=0)) + self.assertTrue(col_data.is_valid(max_depth=1)) + self.assertFalse(col_data.is_valid(max_depth=2)) + + col_data.clear() + self.assertTrue(col_data.is_valid(max_depth=0)) + self.assertFalse(col_data.is_valid(max_depth=1)) + self.assertFalse(col_data.is_valid(max_depth=2)) + class TestDataBindings(TestDataObjects): diff --git a/xmlschema/dataobjects.py b/xmlschema/dataobjects.py index e838f7be..215a3c76 100644 --- a/xmlschema/dataobjects.py +++ b/xmlschema/dataobjects.py @@ -55,6 +55,8 @@ def __init__(self, tag, value=None, attrib=None, nsmap=None, xsd_element=None, x self.xsd_type = xsd_type or xsd_element.type elif xsd_type is not None: self.xsd_type = xsd_type + elif self.xsd_element is not None: + self._encoder = self.xsd_element def __getitem__(self, i): return self._children[i] @@ -144,19 +146,20 @@ def local_name(self): """The local part of the tag.""" return local_name(self.tag) - def validate(self, use_defaults=True, namespaces=None): + def validate(self, use_defaults=True, namespaces=None, max_depth=None): """ Validates the XML data object. :param use_defaults: whether to use default values for filling missing data. :param namespaces: is an optional mapping from namespace prefix to URI. + :param max_depth: maximum depth for validation, for default there is no limit. :raises: :exc:`XMLSchemaValidationError` if XML data object is not valid. :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. """ - for error in self.iter_errors(use_defaults, namespaces): + for error in self.iter_errors(use_defaults, namespaces, max_depth): raise error - def is_valid(self, use_defaults=True, namespaces=None): + def is_valid(self, use_defaults=True, namespaces=None, max_depth=None): """ Like :meth:`validate` except it does not raise an exception on validation error but returns ``True`` if the XML data object is valid, ``False`` if @@ -164,14 +167,15 @@ def is_valid(self, use_defaults=True, namespaces=None): :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. """ - return next(self.iter_errors(use_defaults, namespaces), None) is None + return next(self.iter_errors(use_defaults, namespaces, max_depth), None) is None - def iter_errors(self, use_defaults=True, namespaces=None): + def iter_errors(self, use_defaults=True, namespaces=None, max_depth=None): """ Generates a sequence of validation errors if the XML data object is invalid. :param use_defaults: whether to use default values for filling missing data. :param namespaces: is an optional mapping from namespace prefix to URI. + :param max_depth: maximum depth for validation, for default there is no limit. :raises: :exc:`XMLSchemaValueError` if the instance has no schema bindings. """ if self._encoder is None: @@ -182,6 +186,8 @@ def iter_errors(self, use_defaults=True, namespaces=None): kwargs['use_defaults'] = False if namespaces: kwargs['namespaces'] = namespaces + if isinstance(max_depth, int) and max_depth >= 0: + kwargs['max_depth'] = max_depth for result in self._encoder.iter_encode(self, **kwargs): if isinstance(result, validators.XMLSchemaValidationError): diff --git a/xmlschema/validators/elements.py b/xmlschema/validators/elements.py index 55ba8116..176c7d52 100644 --- a/xmlschema/validators/elements.py +++ b/xmlschema/validators/elements.py @@ -781,6 +781,8 @@ def iter_encode(self, obj, validation='lax', **kwargs): :return: yields an Element, eventually preceded by a sequence of \ validation or encoding errors. """ + errors = [] + try: converter = kwargs['converter'] except KeyError: @@ -793,10 +795,17 @@ def iter_encode(self, obj, validation='lax', **kwargs): level = kwargs['level'] except KeyError: level = 0 + element_data = converter.element_encode(obj, self, level) + if not self.is_matching(element_data.tag, self.default_namespace): + errors.append("data tag does not match XSD element name") + + if 'max_depth' in kwargs and kwargs['max_depth'] == 0: + for e in errors: + yield self.validation_error(validation, e, **kwargs) + return + else: + element_data = converter.element_encode(obj, self, level) - element_data = converter.element_encode(obj, self, level) - errors = [] - tag = element_data.tag text = None children = element_data.content attributes = () @@ -888,7 +897,7 @@ def iter_encode(self, obj, validation='lax', **kwargs): elif result: text, children = result - elem = converter.etree_element(tag, text, children, attributes, level) + elem = converter.etree_element(element_data.tag, text, children, attributes, level) if errors: for e in errors: diff --git a/xmlschema/validators/groups.py b/xmlschema/validators/groups.py index 8a525ae1..93ce1208 100644 --- a/xmlschema/validators/groups.py +++ b/xmlschema/validators/groups.py @@ -812,6 +812,7 @@ def iter_encode(self, element_data, validation='lax', **kwargs): model = ModelVisitor(self) index = cdata_index = 0 wrong_content_type = False + over_max_depth = 'max_depth' in kwargs and kwargs['max_depth'] <= level if element_data.content is None: content = [] @@ -874,6 +875,9 @@ def iter_encode(self, element_data, validation='lax', **kwargs): yield self.validation_error(validation, reason, value, **kwargs) continue + if over_max_depth: + continue + for result in xsd_element.iter_encode(value, validation, **kwargs): if isinstance(result, XMLSchemaValidationError): yield result From 84a0187501c90060484c8423eaa04c31bfb7ba00 Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Thu, 11 Feb 2021 10:51:51 +0100 Subject: [PATCH 6/7] Rename XsdElement create_binding() to get_binding() - Add also replace_existing=False keyword argument --- CHANGELOG.rst | 10 +++++----- doc/conf.py | 4 ++-- publiccode.yml | 4 ++-- setup.py | 2 +- tests/test_dataobjects.py | 2 +- xmlschema/__init__.py | 12 ++++++------ xmlschema/dataobjects.py | 2 +- xmlschema/validators/elements.py | 23 +++++++++++++++-------- xmlschema/validators/global_maps.py | 2 +- xmlschema/validators/schema.py | 9 +++++++-- 10 files changed, 41 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 09890e00..68b2c7f8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,11 +2,11 @@ CHANGELOG ********* -`v1.6.0`_ (2021-02-05) +`v1.5.1`_ (2021-02-11) ====================== -* TODO: Add DataElementMeta metaclass for customizing DataElement subclasses -* TODO: Implement PythonGenerator with three set of templates (xmlschema API - based, simple class hierarchy, dataclass hierarchy) +* Optimize NamespaceView read-only mapping +* Add experimental XML data bindings with a DataBindingConverter +* Add exprimental PythonGenerator for static codegen with Jinja2 `v1.5.0`_ (2021-02-05) ====================== @@ -408,4 +408,4 @@ v0.9.6 (2017-05-05) .. _v1.4.1: https://github.com/brunato/xmlschema/compare/v1.4.0...v1.4.1 .. _v1.4.2: https://github.com/brunato/xmlschema/compare/v1.4.1...v1.4.2 .. _v1.5.0: https://github.com/brunato/xmlschema/compare/v1.4.2...v1.5.0 -.. _v1.6.0: https://github.com/brunato/xmlschema/compare/v1.5.0...v1.6.0 +.. _v1.6.0: https://github.com/brunato/xmlschema/compare/v1.5.0...v1.5.1 diff --git a/doc/conf.py b/doc/conf.py index 262e0d35..df2e1f49 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -63,9 +63,9 @@ # built documents. # # The short X.Y version. -version = '1.6' +version = '1.5' # The full version, including alpha/beta/rc tags. -release = '1.6.0' +release = '1.5.1' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/publiccode.yml b/publiccode.yml index 7886daf8..20cf9c87 100644 --- a/publiccode.yml +++ b/publiccode.yml @@ -6,8 +6,8 @@ publiccodeYmlVersion: '0.2' name: xmlschema url: 'https://github.com/sissaschool/xmlschema' landingURL: 'https://github.com/sissaschool/xmlschema' -releaseDate: '2021-02-05' -softwareVersion: v1.5.0 +releaseDate: '2021-02-11' +softwareVersion: v1.5.1 developmentStatus: stable platforms: - linux diff --git a/setup.py b/setup.py index 378bf8a1..e05cd748 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ setup( name='xmlschema', - version='1.6.0', + version='1.5.1', packages=find_packages(include=['xmlschema', 'xmlschema.*']), include_package_data=True, entry_points={ diff --git a/tests/test_dataobjects.py b/tests/test_dataobjects.py index 1bc9cb00..498e6271 100644 --- a/tests/test_dataobjects.py +++ b/tests/test_dataobjects.py @@ -286,7 +286,7 @@ def test_element_binding(self): xsd_element.binding = None try: - binding_class = xsd_element.create_binding() + binding_class = xsd_element.get_binding() self.assertEqual(binding_class.__name__, 'CollectionBinding') self.assertEqual(binding_class.__qualname__, 'CollectionBinding') self.assertIsNone(binding_class.__module__) diff --git a/xmlschema/__init__.py b/xmlschema/__init__.py index 5c0548ab..a541cfc6 100644 --- a/xmlschema/__init__.py +++ b/xmlschema/__init__.py @@ -17,7 +17,7 @@ from .converters import ElementData, XMLSchemaConverter, \ UnorderedConverter, ParkerConverter, BadgerFishConverter, \ AbderaConverter, JsonMLConverter, ColumnarConverter -from .dataobjects import DataElement, DataElementConverter +from .dataobjects import DataElement, DataElementConverter, DataBindingConverter from .documents import validate, is_valid, iter_errors, to_dict, to_json, \ from_json, XmlDocument @@ -30,7 +30,7 @@ XsdComponent, XsdType, XsdElement, XsdAttribute ) -__version__ = '1.6.0' +__version__ = '1.5.1' __author__ = "Davide Brunato" __contact__ = "brunato@sissa.it" __copyright__ = "Copyright 2016-2021, SISSA" @@ -42,10 +42,10 @@ 'limits', 'XMLSchemaException', 'XMLResourceError', 'XMLSchemaNamespaceError', 'etree_tostring', 'normalize_url', 'normalize_locations', 'fetch_resource', 'fetch_namespaces', 'fetch_schema_locations', 'fetch_schema', 'XMLResource', - 'ElementPathMixin', 'ElementData', 'DataElement', 'XMLSchemaConverter', - 'UnorderedConverter', 'ParkerConverter', 'BadgerFishConverter', 'AbderaConverter', - 'JsonMLConverter', 'ColumnarConverter', 'DataElementConverter', 'validate', - 'is_valid', 'iter_errors', 'to_dict', 'to_json', 'from_json', 'XmlDocument', + 'ElementPathMixin', 'ElementData', 'XMLSchemaConverter', 'UnorderedConverter', + 'ParkerConverter', 'BadgerFishConverter', 'AbderaConverter', 'JsonMLConverter', + 'ColumnarConverter', 'DataElement', 'DataElementConverter', 'DataBindingConverter', + 'validate', 'is_valid', 'iter_errors', 'to_dict', 'to_json', 'from_json', 'XmlDocument', 'XMLSchemaValidatorError', 'XMLSchemaParseError', 'XMLSchemaNotBuiltError', 'XMLSchemaModelError', 'XMLSchemaModelDepthError', 'XMLSchemaValidationError', 'XMLSchemaDecodeError', 'XMLSchemaEncodeError', 'XMLSchemaChildrenValidationError', diff --git a/xmlschema/dataobjects.py b/xmlschema/dataobjects.py index 215a3c76..28c29965 100644 --- a/xmlschema/dataobjects.py +++ b/xmlschema/dataobjects.py @@ -391,7 +391,7 @@ class DataBindingConverter(DataElementConverter): XML binding classes. """ def element_decode(self, data, xsd_element, xsd_type=None, level=0): - cls = xsd_element.binding or xsd_element.create_binding(self.data_element_class) + cls = xsd_element.get_binding(self.data_element_class) data_element = cls( tag=data.tag, value=data.text, diff --git a/xmlschema/validators/elements.py b/xmlschema/validators/elements.py index 176c7d52..967a2e0d 100644 --- a/xmlschema/validators/elements.py +++ b/xmlschema/validators/elements.py @@ -379,14 +379,21 @@ def block(self): return self._block return self.schema.block_default - def create_binding(self, *bases, **attrs): - """Create data object binding for XSD element.""" - if not bases: - bases = (dataobjects.DataElement,) - attrs['xsd_element'] = self - class_name = '{}Binding'.format(self.local_name.title().replace('_', '')) - - self.binding = dataobjects.DataBindingMeta(class_name, bases, attrs) + def get_binding(self, *bases, replace_existing=False, **attrs): + """ + Gets data object binding for XSD element, creating a new one if it doesn't exist. + + :param bases: base classes to use for creating the binding class. + :param replace_existing: provide `True` to replace an existing binding class. + :param attrs: attribute and method definitions for the binding class body. + """ + if self.binding is None or replace_existing: + if not bases: + bases = (dataobjects.DataElement,) + attrs['xsd_element'] = self + class_name = '{}Binding'.format(self.local_name.title().replace('_', '')) + self.binding = dataobjects.DataBindingMeta(class_name, bases, attrs) + return self.binding def get_attribute(self, name): diff --git a/xmlschema/validators/global_maps.py b/xmlschema/validators/global_maps.py index d5fc1dd1..388cf479 100644 --- a/xmlschema/validators/global_maps.py +++ b/xmlschema/validators/global_maps.py @@ -379,7 +379,7 @@ def create_bindings(self, *bases, **attrs): """Creates data object bindings for the XSD elements of built schemas.""" for xsd_element in self.iter_components(xsd_classes=XsdElement): if xsd_element.target_namespace != XSD_NAMESPACE: - xsd_element.create_binding(*bases, **attrs) + xsd_element.get_binding(*bases, replace_existing=True, **attrs) def clear_bindings(self): for xsd_element in self.iter_components(xsd_classes=XsdElement): diff --git a/xmlschema/validators/schema.py b/xmlschema/validators/schema.py index 18868a87..94742961 100644 --- a/xmlschema/validators/schema.py +++ b/xmlschema/validators/schema.py @@ -962,9 +962,14 @@ def get_element(self, tag, path=None, namespaces=None): return self.find(path, namespaces) def create_bindings(self, *bases, **attrs): - """Creates data object bindings for XSD elements of the schema.""" + """ + Creates data object bindings for XSD elements of the schema. + + :param bases: base classes to use for creating the binding classes. + :param attrs: attribute and method definitions for the binding classes body. + """ for xsd_element in self.iter_components(xsd_classes=XsdElement): - xsd_element.create_binding(*bases, **attrs) + xsd_element.get_binding(*bases, replace_existing=True, **attrs) def _parse_inclusions(self): """Processes schema document inclusions and redefinitions.""" From 183643c0fecce34f795239e4d52ee00bad2fcad7 Mon Sep 17 00:00:00 2001 From: Davide Brunato Date: Thu, 11 Feb 2021 15:03:05 +0100 Subject: [PATCH 7/7] Add PythonGenerator and a sample template for XSD data bindings --- CHANGELOG.rst | 2 +- README.rst | 2 + doc/api.rst | 8 ++- .../examples/collection/collection.py | 27 ++++++++ tests/test_codegen.py | 61 ++++++++++++------- tox.ini | 7 ++- xmlschema/extras/codegen.py | 60 ++++++++++++++++++ .../extras/templates/python/bindings.py.jinja | 27 ++++++++ .../extras}/templates/python/sample.py.jinja | 0 9 files changed, 168 insertions(+), 26 deletions(-) create mode 100644 tests/test_cases/examples/collection/collection.py create mode 100644 xmlschema/extras/templates/python/bindings.py.jinja rename {tests => xmlschema/extras}/templates/python/sample.py.jinja (100%) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 68b2c7f8..83984bd5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,7 +6,7 @@ CHANGELOG ====================== * Optimize NamespaceView read-only mapping * Add experimental XML data bindings with a DataBindingConverter -* Add exprimental PythonGenerator for static codegen with Jinja2 +* Add experimental PythonGenerator for static codegen with Jinja2 `v1.5.0`_ (2021-02-05) ====================== diff --git a/README.rst b/README.rst index d28d6e7e..5010e02c 100644 --- a/README.rst +++ b/README.rst @@ -51,6 +51,8 @@ This library includes the following features: * An XPath based API for finding schema's elements and attributes * Support of XSD validation modes *strict*/*lax*/*skip* * Remote attacks protection by default using an XMLParser that forbids entities +* XML data bindings based on DataElement class (experimental) +* Static code generation with Jinja2 templates (experimental) Installation diff --git a/doc/api.rst b/doc/api.rst index b1d430a6..5861e005 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -132,6 +132,8 @@ The base class `XMLSchemaConverter` is used for defining generic converters. The subclasses implement some of the most used `conventions for converting XML to JSON data `_. +.. autoclass:: xmlschema.ElementData + .. autoclass:: xmlschema.XMLSchemaConverter .. autoattribute:: lossy @@ -159,14 +161,15 @@ to JSON data `_. .. autoclass:: xmlschema.ColumnarConverter -.. autoclass:: xmlschema.DataElementConverter +.. _data-objects-api: Data objects API ================ -.. autoclass:: xmlschema.ElementData .. autoclass:: xmlschema.DataElement +.. autoclass:: xmlschema.DataElementConverter +.. autoclass:: xmlschema.DataBindingConverter .. _xml-resource-api: @@ -405,7 +408,6 @@ Code generators .. automethod:: render_to_files - .. autoclass:: xmlschema.extras.codegen.PythonGenerator diff --git a/tests/test_cases/examples/collection/collection.py b/tests/test_cases/examples/collection/collection.py new file mode 100644 index 00000000..327a0bec --- /dev/null +++ b/tests/test_cases/examples/collection/collection.py @@ -0,0 +1,27 @@ +# +# Copyright (c), 2016-2021, SISSA (International School for Advanced Studies). +# All rights reserved. +# This file is distributed under the terms of the MIT License. +# See the file 'LICENSE' in the root directory of the present +# distribution, or http://opensource.org/licenses/MIT. +# +# Auto-generated code: don't edit this file +# +""" +Sample of XML data bindings for schema collection.xsd +""" +from xmlschema import XMLSchema, DataElement +from xmlschema.dataobjects import DataBindingMeta + +__NAMESPACE__ = "http://example.com/ns/collection" + +schema = XMLSchema("collection.xsd") + + +class CollectionBinding(DataElement, metaclass=DataBindingMeta): + xsd_element = schema.elements['collection'] + + +class PersonBinding(DataElement, metaclass=DataBindingMeta): + xsd_element = schema.elements['person'] + diff --git a/tests/test_codegen.py b/tests/test_codegen.py index b20dcce1..9b0b971f 100644 --- a/tests/test_codegen.py +++ b/tests/test_codegen.py @@ -11,11 +11,15 @@ """Tests concerning XSD based code generators. Requires jinja2 optional dependency.""" import unittest +import os import datetime import ast import platform +import importlib.util from pathlib import Path from textwrap import dedent +from xml.etree import ElementTree + from xmlschema import XMLSchema10, XMLSchema11 from xmlschema.names import XSD_ANY_TYPE, XSD_STRING, XSD_FLOAT @@ -28,7 +32,7 @@ PythonGenerator = None DemoGenerator = None else: - from xmlschema.extras.codegen import filter_method, AbstractGenerator + from xmlschema.extras.codegen import filter_method, AbstractGenerator, PythonGenerator class DemoGenerator(AbstractGenerator): formal_language = 'Demo' @@ -68,26 +72,6 @@ def not_an_instance_filter(self): return - class PythonGenerator(AbstractGenerator): - """ - Python code sample generator for XSD schemas. - """ - formal_language = 'Python' - - searchpaths = ['templates/python/'] - - builtin_types = { - 'string': 'str', - 'boolean': 'bool', - 'float': 'float', - 'double': 'float', - 'integer': 'int', - 'unsignedByte': 'int', - 'nonNegativeInteger': 'int', - 'positiveInteger': 'int', - } - - def casepath(relative_path): return str(Path(__file__).absolute().parent.joinpath('test_cases', relative_path)) @@ -347,6 +331,14 @@ def test_language_type_filter(self): self.generator.render('python_type_filter_test.jinja'), ['str'] ) + def test_list_templates(self): + template_dir = Path(__file__).parent.joinpath('templates') + language = self.generator_class.formal_language.lower() + + templates = {'sample.py.jinja', 'bindings.py.jinja'} + templates.update(x.name for x in template_dir.glob('filters/*'.format(language))) + self.assertSetEqual(set(self.generator.list_templates()), templates) + def test_sample_module(self): generator = PythonGenerator(self.col_xsd_file) @@ -354,6 +346,33 @@ def test_sample_module(self): ast_module = ast.parse(python_module) self.assertIsInstance(ast_module, ast.Module) + def test_bindings_module(self): + generator = PythonGenerator(self.col_xsd_file) + + python_module = generator.render('bindings.py.jinja')[0] + + ast_module = ast.parse(python_module) + self.assertIsInstance(ast_module, ast.Module) + + collection_dir = Path(__file__).parent.joinpath('test_cases/examples/collection') + cwd = os.getcwd() + try: + os.chdir(str(collection_dir)) + with open('collection.py', 'w') as fp: + fp.write(python_module) + + spec = importlib.util.spec_from_file_location('collection', 'collection.py') + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + except Exception: + pass + else: + col_data = module.CollectionBinding.fromsource('collection.xml') + col_root = ElementTree.XML(col_data.tostring()) + self.assertEqual(col_root.tag, '{http://example.com/ns/collection}collection') + finally: + os.chdir(cwd) + if __name__ == '__main__': import platform diff --git a/tox.ini b/tox.ini index 65521831..3ef6ded4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ # and then run "tox" from this directory. [tox] -envlist = py{36,37,38,39}, pypy3, ep212, ep213, docs, flake8, coverage +envlist = py{36,37,38,39}, pypy3, ep21{2,3,4}, docs, flake8, coverage skip_missing_interpreters = true toxworkdir = {homedir}/.tox/xmlschema @@ -46,6 +46,11 @@ deps = elementpath==2.1.3 lxml +[testenv:ep214] +deps = + elementpath==2.1.4 + lxml + [testenv:docs] commands = make -C doc html diff --git a/xmlschema/extras/codegen.py b/xmlschema/extras/codegen.py index c437548c..97c1721a 100644 --- a/xmlschema/extras/codegen.py +++ b/xmlschema/extras/codegen.py @@ -548,3 +548,63 @@ def multi_sequence(xsd_type): if xsd_type.has_simple_content(): return False return any(e.is_multiple() for e in xsd_type.content_type.iter_elements()) + + +class PythonGenerator(AbstractGenerator): + """A Python code generator for XSD schemas.""" + + formal_language = 'Python' + + searchpaths = ['templates/python/'] + + builtin_types = { + 'string': 'str', + 'decimal': 'decimal.Decimal', + 'float': 'float', + 'double': 'float', + + 'date': 'datatypes.Date10', + 'dateTime': 'datatypes.DateTime10', + 'gDay': 'datatypes.GregorianDay', + 'gMonth': 'datatypes.GregorianMonth', + 'gMonthDay': 'datatypes.GregorianMonthDay', + 'gYear': 'datatypes.GregorianYear10', + 'gYearMonth': 'datatypes.GregorianYearMonth10', + 'time': 'datatypes.Time', + 'duration': 'datatypes.Duration', + + 'QName': 'datatypes.QName', + 'NOTATION': 'datatypes.DateTime10', + 'anyURI': 'datatypes.AnyURI', + 'boolean': 'bool', + 'base64Binary': 'datatypes.Base64Binary', + 'hexBinary': 'datatypes.HexBinary', + 'normalizedString': 'str', + 'token': 'str', + 'language': 'str', + 'Name': 'str', + 'NCName': 'str', + 'ID': 'str', + 'IDREF': 'str', + 'ENTITY': 'str', + 'NMTOKEN': 'str', + + 'integer': 'int', + 'long': 'int', + 'int': 'int', + 'short': 'int', + 'byte': 'int', + 'nonNegativeInteger': 'int', + 'positiveInteger': 'int', + 'unsignedLong': 'int', + 'unsignedInt': 'int', + 'unsignedShort': 'int', + 'unsignedByte': 'int', + 'nonPositiveInteger': 'int', + 'negativeInteger': 'int', + + # XSD 1.1 built-in types + 'dateTimeStamp': 'datatypes.DateTimeStamp10', + 'dayTimeDuration': 'datatypes.DayTimeDuration', + 'yearMonthDuration': 'datatypes.YearMonthDuration', + } diff --git a/xmlschema/extras/templates/python/bindings.py.jinja b/xmlschema/extras/templates/python/bindings.py.jinja new file mode 100644 index 00000000..57e9f74b --- /dev/null +++ b/xmlschema/extras/templates/python/bindings.py.jinja @@ -0,0 +1,27 @@ +# +# Copyright (c), 2016-2021, SISSA (International School for Advanced Studies). +# All rights reserved. +# This file is distributed under the terms of the MIT License. +# See the file 'LICENSE' in the root directory of the present +# distribution, or http://opensource.org/licenses/MIT. +# +# Auto-generated code: don't edit this file +# +""" +Sample of XML data bindings for schema {{ schema.name }} +""" +from xmlschema import XMLSchema, DataElement +from xmlschema.dataobjects import DataBindingMeta + +{% if schema.target_namespace -%} +__NAMESPACE__ = "{{ schema.target_namespace }}" +{%- endif %} + +schema = XMLSchema("{{ schema.name }}") + +{# Bindings for global elements #} +{%- for xsd_element in schema.elements.values() %} +class {{ xsd_element|name|capitalize }}Binding(DataElement, metaclass=DataBindingMeta): + xsd_element = schema.elements['{{ xsd_element.local_name }}'] + +{% endfor %} \ No newline at end of file diff --git a/tests/templates/python/sample.py.jinja b/xmlschema/extras/templates/python/sample.py.jinja similarity index 100% rename from tests/templates/python/sample.py.jinja rename to xmlschema/extras/templates/python/sample.py.jinja