Merge pull request #727 from IanCa/develop_new

Updates to schema loading and comparison

hed/schema/hed_schema.py

@@ -1,12 +1,10 @@
-import os
-import shutil
 import json
 
 from hed.schema.hed_schema_constants import HedKey, HedSectionKey
 from hed.schema import hed_schema_constants as constants
 from hed.schema.schema_io import schema_util
-from hed.schema.schema_io.schema2xml import HedSchema2XML
-from hed.schema.schema_io.schema2wiki import HedSchema2Wiki
+from hed.schema.schema_io.schema2xml import Schema2XML
+from hed.schema.schema_io.schema2wiki import Schema2Wiki
 from hed.schema.hed_schema_section import HedSchemaSection, HedSchemaTagSection, HedSchemaUnitClassSection
 from hed.errors import ErrorHandler
 from hed.errors.error_types import ValidationErrors
@@ -212,8 +210,7 @@ def get_as_mediawiki_string(self, save_merged=False):
             str:  The schema as a string in mediawiki format.
 
         """
-        schema2wiki = HedSchema2Wiki()
-        output_strings = schema2wiki.process_schema(self, save_merged)
+        output_strings = Schema2Wiki.process_schema(self, save_merged)
         return '\n'.join(output_strings)
 
     def get_as_xml_string(self, save_merged=True):
@@ -227,12 +224,11 @@ def get_as_xml_string(self, save_merged=True):
             str: Return the schema as an XML string.
 
         """
-        schema2xml = HedSchema2XML()
-        xml_tree = schema2xml.process_schema(self, save_merged)
+        xml_tree = Schema2XML.process_schema(self, save_merged)
         return schema_util._xml_element_2_str(xml_tree)
 
     def save_as_mediawiki(self, filename=None, save_merged=False):
-        """ Save as mediawiki to a temporary file.
+        """ Save as mediawiki to a file.
 
         filename: str
             If present, move the resulting file to this location.
@@ -243,19 +239,12 @@ def save_as_mediawiki(self, filename=None, save_merged=False):
         Returns:
             str:    The newly created schema filename.
         """
-        schema2wiki = HedSchema2Wiki()
-        output_strings = schema2wiki.process_schema(self, save_merged)
+        output_strings = Schema2Wiki.process_schema(self, save_merged)
         local_wiki_file = schema_util.write_strings_to_file(output_strings, ".mediawiki")
-        if filename:
-            directory = os.path.dirname(filename)
-            if directory and not os.path.exists(directory):
-                os.makedirs(directory)
-            shutil.move(local_wiki_file, filename)
-            return filename
-        return local_wiki_file
+        return schema_util.move_file(local_wiki_file, filename)
 
     def save_as_xml(self, filename=None, save_merged=True):
-        """ Save as XML to a temporary file.
+        """ Save as XML to a file.
 
         filename: str
             If present, move the resulting file to this location.
@@ -266,16 +255,9 @@ def save_as_xml(self, filename=None, save_merged=True):
         Returns:
             str: The name of the newly created schema file.
         """
-        schema2xml = HedSchema2XML()
-        xml_tree = schema2xml.process_schema(self, save_merged)
+        xml_tree = Schema2XML.process_schema(self, save_merged)
         local_xml_file = schema_util.write_xml_tree_2_xml_file(xml_tree, ".xml")
-        if filename:
-            directory = os.path.dirname(filename)
-            if directory and not os.path.exists(directory):
-                os.makedirs(directory)
-            shutil.move(local_xml_file, filename)
-            return filename
-        return local_xml_file
+        return schema_util.move_file(local_xml_file, filename)
 
     def set_schema_prefix(self, schema_namespace):
         """ Set library namespace associated for this schema.
@@ -746,8 +728,9 @@ def _get_attributes_for_section(self, key_class):
     # Semi private function used to create a schema in memory(usually from a source file)
     # ===============================================
     def _add_tag_to_dict(self, long_tag_name, new_entry, key_class):
-        # No reason we can't add this here always
-        if self.library and not self.merged and self.with_standard:
+        # Add the InLibrary attribute to any library schemas as they are loaded
+        # These are later removed when they are saved out, if saving unmerged
+        if self.library and (not self.with_standard or (not self.merged and self.with_standard)):
             new_entry._set_attribute_value(HedKey.InLibrary, self.library)
 
         section = self._sections[key_class]

hed/schema/hed_schema_constants.py

@@ -1,5 +1,6 @@
 from enum import Enum
 
+
 class HedSectionKey(Enum):
     """ Kegs designating specific sections in a HedSchema object.
     """
@@ -69,4 +70,16 @@ class HedKey:
 VERSION_ATTRIBUTE = 'version'
 LIBRARY_ATTRIBUTE = 'library'
 WITH_STANDARD_ATTRIBUTE = "withStandard"
-UNMERGED_ATTRIBUTE = "unmerged"
+UNMERGED_ATTRIBUTE = "unmerged"
+NS_ATTRIB = "xmlns:xsi"
+NO_LOC_ATTRIB = "xsi:noNamespaceSchemaLocation"
+
+# A list of all attributes that can appear in the header line
+valid_header_attributes = {
+    VERSION_ATTRIBUTE,
+    LIBRARY_ATTRIBUTE,
+    WITH_STANDARD_ATTRIBUTE,
+    NS_ATTRIB,
+    NO_LOC_ATTRIB,
+    UNMERGED_ATTRIBUTE
+}

hed/schema/hed_schema_io.py

@@ -2,9 +2,9 @@
 import os
 import json
 import functools
-from hed.schema.schema_io.xml2schema import HedSchemaXMLParser
-from hed.schema.schema_io.wiki2schema import HedSchemaWikiParser
-from hed.schema import hed_schema_constants, hed_cache
+from hed.schema.schema_io.xml2schema import SchemaLoaderXML
+from hed.schema.schema_io.wiki2schema import SchemaLoaderWiki
+from hed.schema import hed_cache
 
 from hed.errors.exceptions import HedFileError, HedExceptions
 from hed.schema.schema_io import schema_util
@@ -39,9 +39,9 @@ def from_string(schema_string, file_type=".xml", schema_namespace=None):
                            filename=schema_string)
 
     if file_type.endswith(".xml"):
-        hed_schema = HedSchemaXMLParser.load_xml(schema_as_string=schema_string)
+        hed_schema = SchemaLoaderXML.load(schema_as_string=schema_string)
     elif file_type.endswith(".mediawiki"):
-        hed_schema = HedSchemaWikiParser.load_wiki(schema_as_string=schema_string)
+        hed_schema = SchemaLoaderWiki.load(schema_as_string=schema_string)
     else:
         raise HedFileError(HedExceptions.INVALID_EXTENSION, "Unknown schema extension", filename=file_type)
 
@@ -77,9 +77,9 @@ def load_schema(hed_path=None, schema_namespace=None):
         file_as_string = schema_util.url_to_string(hed_path)
         hed_schema = from_string(file_as_string, file_type=os.path.splitext(hed_path.lower())[1])
     elif hed_path.lower().endswith(".xml"):
-        hed_schema = HedSchemaXMLParser.load_xml(hed_path)
+        hed_schema = SchemaLoaderXML.load(hed_path)
     elif hed_path.lower().endswith(".mediawiki"):
-        hed_schema = HedSchemaWikiParser.load_wiki(hed_path)
+        hed_schema = SchemaLoaderWiki.load(hed_path)
     else:
         raise HedFileError(HedExceptions.INVALID_EXTENSION, "Unknown schema extension", filename=hed_path)
 
@@ -89,7 +89,7 @@ def load_schema(hed_path=None, schema_namespace=None):
     return hed_schema
 
 
-# todo: this could be updated to also support .mediawiki format.
+# If this is actually used, we could easily add other versions/update this one
 def get_hed_xml_version(xml_file_path):
     """ Get the version number from a HED XML file.
 
@@ -102,8 +102,8 @@ def get_hed_xml_version(xml_file_path):
     :raises HedFileError:
         - There is an issue loading the schema
     """
-    root_node = HedSchemaXMLParser._parse_hed_xml(xml_file_path)
-    return root_node.attrib[hed_schema_constants.VERSION_ATTRIBUTE]
+    parser = SchemaLoaderXML(xml_file_path)
+    return parser.schema.version
 
 
 @functools.lru_cache(maxsize=MAX_MEMORY_CACHE)
@@ -143,7 +143,9 @@ def _load_schema_version(xml_version=None, xml_folder=None):
             hed_cache.cache_xml_versions(cache_folder=xml_folder)
             final_hed_xml_file = hed_cache.get_hed_version_path(xml_version, library_name, xml_folder)
             if not final_hed_xml_file:
-                raise HedFileError(HedExceptions.FILE_NOT_FOUND, f"HED version '{xml_version}' not found in cache: {hed_cache.get_cache_directory()}", filename=xml_folder)
+                raise HedFileError(HedExceptions.FILE_NOT_FOUND,
+                                   f"HED version '{xml_version}' not found in cache: {hed_cache.get_cache_directory()}",
+                                   filename=xml_folder)
             hed_schema = load_schema(final_hed_xml_file)
         else:
             raise e