Merge pull request #900 from IanCa/develop

Implement validation with new 83 schema properties/rules

hed/errors/error_types.py

@@ -133,17 +133,17 @@ class SchemaAttributeErrors:
     SCHEMA_DEPRECATED_INVALID = "SCHEMA_DEPRECATED_INVALID"
     SCHEMA_CHILD_OF_DEPRECATED = "SCHEMA_CHILD_OF_DEPRECATED"
     SCHEMA_ATTRIBUTE_VALUE_DEPRECATED = "SCHEMA_ATTRIBUTE_VALUE_DEPRECATED"
-    SCHEMA_SUGGESTED_TAG_INVALID = "SCHEMA_SUGGESTED_TAG_INVALID"
 
-    SCHEMA_UNIT_CLASS_INVALID = "SCHEMA_UNIT_CLASS_INVALID"
-    SCHEMA_VALUE_CLASS_INVALID = "SCHEMA_VALUE_CLASS_INVALID"
     SCHEMA_ALLOWED_CHARACTERS_INVALID = "SCHEMA_ALLOWED_CHARACTERS_INVALID"
     SCHEMA_IN_LIBRARY_INVALID = "SCHEMA_IN_LIBRARY_INVALID"
 
+    SCHEMA_ATTRIBUTE_NUMERIC_INVALID = "SCHEMA_ATTRIBUTE_NUMERIC_INVALID"
     SCHEMA_DEFAULT_UNITS_INVALID = "SCHEMA_DEFAULT_UNITS_INVALID"
     SCHEMA_DEFAULT_UNITS_DEPRECATED = "SCHEMA_DEFAULT_UNITS_DEPRECATED"
     SCHEMA_CONVERSION_FACTOR_NOT_POSITIVE = "SCHEMA_CONVERSION_FACTOR_NOT_POSITIVE"
 
+    SCHEMA_GENERIC_ATTRIBUTE_VALUE_INVALID = "SCHEMA_GENERIC_ATTRIBUTE_VALUE_INVALID"
+
 
 class DefinitionErrors:
     # These are all DEFINITION_INVALID errors

hed/errors/schema_error_messages.py

@@ -74,22 +74,16 @@ def schema_error_SCHEMA_ATTRIBUTE_VALUE_DEPRECATED(tag, deprecated_suggestion, a
             f"and an alternative method of tagging should be used.")
 
 
-@hed_error(SchemaAttributeErrors.SCHEMA_SUGGESTED_TAG_INVALID,
+@hed_error(SchemaAttributeErrors.SCHEMA_GENERIC_ATTRIBUTE_VALUE_INVALID,
            actual_code=SchemaAttributeErrors.SCHEMA_ATTRIBUTE_VALUE_INVALID)
-def schema_error_SCHEMA_SUGGESTED_TAG_INVALID(suggestedTag, invalidSuggestedTag, attribute_name):
-    return f"Tag '{suggestedTag}' has an invalid {attribute_name}: '{invalidSuggestedTag}'."
+def schema_error_GENERIC_ATTRIBUTE_VALUE_INVALID(tag, invalid_value, attribute_name):
+    return f"Element '{tag}' has an invalid {attribute_name}: '{invalid_value}'."
 
 
-@hed_error(SchemaAttributeErrors.SCHEMA_UNIT_CLASS_INVALID,
+@hed_error(SchemaAttributeErrors.SCHEMA_ATTRIBUTE_NUMERIC_INVALID,
            actual_code=SchemaAttributeErrors.SCHEMA_ATTRIBUTE_VALUE_INVALID)
-def schema_error_SCHEMA_UNIT_CLASS_INVALID(tag, unit_class, attribute_name):
-    return f"Tag '{tag}' has an invalid {attribute_name}: '{unit_class}'."
-
-
-@hed_error(SchemaAttributeErrors.SCHEMA_VALUE_CLASS_INVALID,
-           actual_code=SchemaAttributeErrors.SCHEMA_ATTRIBUTE_VALUE_INVALID)
-def schema_error_SCHEMA_VALUE_CLASS_INVALID(tag, unit_class, attribute_name):
-    return f"Tag '{tag}' has an invalid {attribute_name}: '{unit_class}'."
+def schema_error_SCHEMA_ATTRIBUTE_NUMERIC_INVALID(tag, invalid_value, attribute_name):
+    return f"Element '{tag}' has an invalid {attribute_name}: '{invalid_value}'.  Should be numeric."
 
 
 @hed_error(SchemaAttributeErrors.SCHEMA_DEFAULT_UNITS_INVALID,

hed/schema/hed_schema.py

@@ -1,7 +1,7 @@
 import json
 import os
 
-from hed.schema.hed_schema_constants import HedKey, HedSectionKey
+from hed.schema.hed_schema_constants import HedKey, HedSectionKey, HedKey83
 from hed.schema import hed_schema_constants as constants
 from hed.schema.schema_io import schema_util
 from hed.schema.schema_io.schema2xml import Schema2XML
@@ -635,7 +635,7 @@ def _initialize_attributes(self, key_class):
     # ===============================================
     # Getters used to write out schema primarily.
     # ===============================================
-    def get_tag_attribute_names(self):
+    def get_tag_attribute_names_old(self):
         """ Return a dict of all allowed tag attributes.
 
         Returns:
@@ -648,27 +648,6 @@ def get_tag_attribute_names(self):
                 and not tag_entry.has_attribute(HedKey.UnitModifierProperty)
                 and not tag_entry.has_attribute(HedKey.ValueClassProperty)}
 
-    def get_all_tag_attributes(self, tag_name, key_class=HedSectionKey.Tags):
-        """ Gather all attributes for a given tag name.
-
-        Parameters:
-            tag_name (str): The name of the tag to check.
-            key_class (str): The type of attributes requested.  e.g. Tag, Units, Unit modifiers, or attributes.
-
-        Returns:
-            dict: A dictionary of attribute name and attribute value.
-
-        Notes:
-            If keys is None, gets all normal hed tag attributes.
-
-        """
-        tag_entry = self._get_tag_entry(tag_name, key_class)
-        attributes = {}
-        if tag_entry:
-            attributes = tag_entry.attributes
-
-        return attributes
-
     # ===============================================
     # Private utility functions
     # ===============================================
@@ -717,47 +696,58 @@ def _get_modifiers_for_unit(self, unit):
         valid_modifiers = self.unit_modifiers.get_entries_with_attribute(modifier_attribute_name)
         return valid_modifiers
 
-    def _add_element_property_attributes(self, attribute_dict):
+    def _add_element_property_attributes(self, attribute_dict, attribute_name):
         attributes = {attribute: entry for attribute, entry in self._sections[HedSectionKey.Attributes].items()
-                      if entry.has_attribute(HedKey.ElementProperty)}
+                      if entry.has_attribute(attribute_name)}
 
         attribute_dict.update(attributes)
 
     def _get_attributes_for_section(self, key_class):
-        """ Return the valid attributes for this section.
+        """Return the valid attributes for this section.
 
         Parameters:
             key_class (HedSectionKey): The HedKey for this section.
 
         Returns:
-            dict or HedSchemaSection: A dict of all the attributes and this section.
-
+            dict: A dict of all the attributes for this section.
         """
-        if key_class == HedSectionKey.Tags:
-            return self.get_tag_attribute_names()
-        elif key_class == HedSectionKey.Attributes:
-            prop_added_dict = {key: value for key, value in self._sections[HedSectionKey.Properties].items()}
-            self._add_element_property_attributes(prop_added_dict)
-            return prop_added_dict
-        elif key_class == HedSectionKey.Properties:
+        element_prop_key = HedKey83.ElementDomain if self.schema_83_props else HedKey.ElementProperty
+
+        # Common logic for Attributes and Properties
+        if key_class in [HedSectionKey.Attributes, HedSectionKey.Properties]:
             prop_added_dict = {}
-            self._add_element_property_attributes(prop_added_dict)
+            if key_class == HedSectionKey.Attributes:
+                prop_added_dict = {key: value for key, value in self._sections[HedSectionKey.Properties].items()}
+            self._add_element_property_attributes(prop_added_dict, element_prop_key)
             return prop_added_dict
+
+        if self.schema_83_props:
+            attrib_classes = {
+                HedSectionKey.UnitClasses: HedKey83.UnitClassDomain,
+                HedSectionKey.Units: HedKey83.UnitDomain,
+                HedSectionKey.UnitModifiers: HedKey83.UnitModifierDomain,
+                HedSectionKey.ValueClasses: HedKey83.ValueClassDomain,
+                HedSectionKey.Tags: HedKey83.TagDomain
+            }
         else:
             attrib_classes = {
                 HedSectionKey.UnitClasses: HedKey.UnitClassProperty,
                 HedSectionKey.Units: HedKey.UnitProperty,
                 HedSectionKey.UnitModifiers: HedKey.UnitModifierProperty,
                 HedSectionKey.ValueClasses: HedKey.ValueClassProperty
             }
-            attrib_class = attrib_classes.get(key_class, None)
-            if attrib_class is None:
-                return []
+            if key_class == HedSectionKey.Tags:
+                return self.get_tag_attribute_names_old()
 
-            attributes = {attribute: entry for attribute, entry in self._sections[HedSectionKey.Attributes].items()
-                          if entry.has_attribute(attrib_class) or entry.has_attribute(HedKey.ElementProperty)}
-            return attributes
+        # Retrieve attributes based on the determined class
+        attrib_class = attrib_classes.get(key_class)
+        if not attrib_class:
+            return []
 
+        attributes = {attribute: entry for attribute, entry in self._sections[HedSectionKey.Attributes].items()
+                      if entry.has_attribute(attrib_class) or entry.has_attribute(element_prop_key)}
+        return attributes
+
     # ===============================================
     # Semi private function used to create a schema in memory(usually from a source file)
     # ===============================================

hed/schema/hed_schema_base.py

@@ -3,6 +3,7 @@
 """
 from hed.schema.hed_schema_constants import HedSectionKey
 from abc import ABC, abstractmethod
+from hed.schema.schema_io import schema_util
 
 
 class HedSchemaBase(ABC):
@@ -12,6 +13,7 @@ class HedSchemaBase(ABC):
     """
     def __init__(self):
         self._name = ""  # User provided identifier for this schema(not used for equality comparison or saved)
+        self._schema83 = None  # If True, this is an 8.3 style schema for validation/attribute purposes
         pass
 
     @property
@@ -25,6 +27,17 @@ def name(self):
     def name(self, name):
         self._name = name
 
+    @property
+    def schema_83_props(self):
+        """Returns if this is an 8.3.0 or greater schema.
+
+        Returns:
+            is_83_schema(bool): True if standard or partnered schema is 8.3.0 or greater."""
+        if self._schema83 is not None:
+            return self._schema83
+
+        self._schema83 = schema_util.schema_version_greater_equal(self, "8.3.0")
+
     @abstractmethod
     def get_schema_versions(self):
         """ A list of HED version strings including namespace and library name if any of this schema.

hed/schema/hed_schema_constants.py

@@ -71,6 +71,26 @@ class HedKey:
     IsInheritedProperty = 'isInheritedProperty'
 
 
+class HedKey83:
+    UnitClassDomain = "unitClassDomain"
+    UnitDomain = "unitDomain"
+    UnitModifierDomain = "unitModifierDomain"
+    ValueClassDomain = "valueClassDomain"
+    ElementDomain = "elementDomain"
+    TagDomain = "tagDomain"
+    AnnotationProperty = "annotationProperty"
+
+    BoolRange = "boolRange"
+
+    # Fully new below this
+    TagRange = "tagRange"
+    NumericRange = "numericRange"
+    StringRange = "stringRange"
+    UnitClassRange = "unitClassRange"
+    UnitRange = "unitRange"
+    ValueClassRange = "valueClassRange"
+
+
 VERSION_ATTRIBUTE = 'version'
 LIBRARY_ATTRIBUTE = 'library'
 WITH_STANDARD_ATTRIBUTE = "withStandard"

hed/schema/hed_schema_entry.py

@@ -318,15 +318,14 @@ def _check_inherited_attribute_internal(self, attribute):
 
         return attribute_values
 
-    def _check_inherited_attribute(self, attribute, return_value=False, return_union=False):
+    def _check_inherited_attribute(self, attribute, return_value=False):
         """
         Checks for the existence of an attribute in this entry and its parents.
 
         Parameters:
             attribute (str): The attribute to check for.
             return_value (bool): If True, returns the actual value of the attribute.
                                  If False, returns a boolean indicating the presence of the attribute.
-            return_union(bool): If True, return a union of all parent values.
 
         Returns:
             bool or any: Depending on the flag return_value,
@@ -335,15 +334,17 @@ def _check_inherited_attribute(self, attribute, return_value=False, return_union
         Notes:
             - The existence of an attribute does not guarantee its validity.
             - For string attributes, the values are joined with a comma as a delimiter from all ancestors.
+            - For other attributes, only the value closest to the leaf is returned
         """
         attribute_values = self._check_inherited_attribute_internal(attribute)
 
         if return_value:
             if not attribute_values:
                 return None
-            if return_union:
+            try:
                 return ",".join(attribute_values)
-            return attribute_values[0]
+            except TypeError:
+                return attribute_values[0]  # Return the lowest level attribute if we don't want the union
         return bool(attribute_values)
 
     def base_tag_has_attribute(self, tag_attribute):
@@ -397,8 +398,7 @@ def _finalize_inherited_attributes(self):
         self.inherited_attributes = self.attributes.copy()
         for attribute in self._section.inheritable_attributes:
             if self._check_inherited_attribute(attribute):
-                treat_as_string = not self.attribute_has_property(attribute, HedKey.BoolProperty)
-                self.inherited_attributes[attribute] = self._check_inherited_attribute(attribute, True, treat_as_string)
+                self.inherited_attributes[attribute] = self._check_inherited_attribute(attribute, True)
 
     def finalize_entry(self, schema):
         """ Called once after schema loading to set state.

hed/schema/hed_schema_section.py

@@ -1,5 +1,5 @@
 from hed.schema.hed_schema_entry import HedSchemaEntry, UnitClassEntry, UnitEntry, HedTagEntry
-from hed.schema.hed_schema_constants import HedSectionKey, HedKey
+from hed.schema.hed_schema_constants import HedSectionKey, HedKey, HedKey83
 
 entries_by_section = {
     HedSectionKey.Properties: HedSchemaEntry,
@@ -254,8 +254,12 @@ def _group_by_top_level_tag(divide_list):
     def _finalize_section(self, hed_schema):
         # Find the attributes with the inherited property
         attribute_section = hed_schema.attributes
-        self.inheritable_attributes = [name for name, value in attribute_section.items()
-                                       if value.has_attribute(HedKey.IsInheritedProperty)]
+        if hed_schema.schema_83_props:
+            self.inheritable_attributes = [name for name, value in attribute_section.items()
+                                           if not value.has_attribute(HedKey83.AnnotationProperty)]
+        else:
+            self.inheritable_attributes = [name for name, value in attribute_section.items()
+                                           if value.has_attribute(HedKey.IsInheritedProperty)]
 
         # Hardcode in extension allowed as it is critical for validation in older schemas
         if not self.inheritable_attributes: