Update for 30043

- add handling to semanticHash for when concepts and other parameters don't exist in filing. (typically ESEF) - 0568591 - fix situational None return in semanticHash - 4b9b951 - refactor XuleValue format_value method to correctly include line number start rules - Xendr fix line numbers (#78) missing fact (#79) - refactor XuleModelIndexer.py and XuleProcessor.py to list the cube properties in the model index for factsets. - refactor TABLE_INDEX_PROPERTIES to be a set instead of a dictionary. - Xodel fix attribute filtering in extract_rel_info and correct typo in document_sort exception message (#81)
xbrlus · Dec 11, 2024 · a2499e0 · a2499e0
1 parent 5f85700
commit a2499e0
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 30 deletions.
diff --git a/plugin/semanticHash.py b/plugin/semanticHash.py
@@ -47,6 +47,7 @@ def semanticHashOptions(parser):
     parserGroup.add_option("--semantic-hash-string", 
                        action="store", 
                        help=_("File name to save the canonicalized string"))
+
 
 def semanticHashUtilityRun(cntlr, options, *arg, **kwargs):
     # save the controller and the options for use later.
@@ -244,19 +245,26 @@ def semanticHashFactValue(fact):
     original_text_block_qname = qname('http://www.xbrl.org/dtr/type/non-numeric', 'textBlockItemType')
     new_text_block_qname = qname('http://www.xbrl.org/dtr/type/2020-01-21', 'textBlockItemType')
     fraction_qname = qname('http://www.xbrl.org/2003/instance', 'fractionItemType')
-    if fact.concept.instanceOfType(original_text_block_qname) or fact.concept.instanceOfType(new_text_block_qname):
+
+    # Unable to identify the concept (fact.concept is likely None)
+    if fact.concept is None:
+        base_type = 'string'
+    elif fact.concept.instanceOfType(original_text_block_qname) or fact.concept.instanceOfType(new_text_block_qname):
         base_type = 'textBlock'
     elif fact.concept.instanceOfType(fraction_qname):
         base_type = 'fraction'
     else:
         base_type = fact.concept.baseXsdType
 
+
     if base_type == 'QName':
         string_value = (fact.xValue.namespaceURI, fact.xValue.localName)
     elif base_type == 'textBlock':
         string_value = fact
     elif base_type == 'fraction':
         string_value = fact
+    elif base_type == 'string':
+        string_value = str(fact.value)
     else:
         string_value = fact.value
 
@@ -281,12 +289,21 @@ def canonicalizeValue(base_type, string_value):
     elif base_type == 'fraction':
         return_value = value_string_hasher(string_value)
     else:
-        # This re will collapse whitespaces
-        return_value = value_string_hasher(re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' '))
+        try: 
+            # This re will collapse whitespaces
+            return_value = value_string_hasher(re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' '))
+        except SemanticHashException:
+            return_value = re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' ')
+
+
+
 
     return return_value
 
 def semanticFormat(name, str_val):
+    if str(type(str_val)) != "<class 'str'>":
+        str_val = ''
+
     return "{}{}.{}".format(name.upper(), len(str_val), str_val)
 
 def UOMDefault(unit):
@@ -301,7 +318,7 @@ def UOMDefault(unit):
 
 def canonicalizeTypedDimensionMember(member):
     element = member.modelXbrl.qnameConcepts.get(member.elementQname)
-    base_type = element.baseXsdType
+    base_type = element.baseXsdType if element is not None else 'string'
     if base_type == 'QName':
         string_value = (member.xValue.namespaceURI, member.xValue.localName)
     else:
@@ -437,7 +454,12 @@ def canonicalizeTextBlock(content):
     # The wrapper allows fragments that don't start with a tag to be parsed. i.e. "abc <div>this is in the div</div>". This
     # example would not parse becasue it starts with text and not a tag.
     wrapper = '<top xmlns="http://www.w3.org/1999/xhtml">{}</top>'.format(content.value.strip())
-    xml_content = etree.fromstring(wrapper)
+    try:
+        p = etree.XMLParser(recover=True)
+        xml_content = etree.XML(wrapper, parser=p)
+    except Exception as ex:
+        raise SemanticHashException("XMLError", 
+                            _("The error is: " + str(ex) + "||" + wrapper))
 
     return_value =  canonicalizeXML(xml_content, include_node_tag=False)
     return return_value

diff --git a/plugin/xendr/xendrRun.py b/plugin/xendr/xendrRun.py
@@ -285,7 +285,7 @@ def substitute_rule(rule_name, sub_info, line_number_subs, rule_results, templat
             # parent_classes = []
             if is_actual_fact(json_result, modelXbrl):
                 # get modelFact
-                if json_result['fact'] is not None:
+                if json_result.get('fact') is not None:
                     model_fact = get_model_object(json_result['fact'], cntlr)
 
 
@@ -1277,7 +1277,14 @@ def render_report(cntlr, options, modelXbrl, *args, **kwargs):
                 rule_meta_data = {'substitutions': meta_substitutions,
                                     'line-numbers': meta_line_number_subs}
 
-            rule_results = run_xule_rules(cntlr, options, modelXbrl, ts, rule_meta_data['standard-rules'] + list(rule_meta_data['showifs'].keys()), catalog_item['xule-rule-set'])
+            # Gather line number start rules
+            line_number_rules = set()
+            for line_number_info in rule_meta_data['line-numbers'].values():
+                for line_number_item in line_number_info:
+                    if 'start-rule' in line_number_item:
+                        line_number_rules.add(line_number_item['start-rule'])
+
+            rule_results = run_xule_rules(cntlr, options, modelXbrl, ts, rule_meta_data['standard-rules'] + list(rule_meta_data['showifs'].keys()) + list(line_number_rules), catalog_item['xule-rule-set'])
 
             # Substitute template
             template_number += 1

diff --git a/plugin/xodel/arelleHelper.py b/plugin/xodel/arelleHelper.py
@@ -303,7 +303,7 @@ def extract_rel_info(model_rel, dts):
     rel_info['type'] = resolve_clark_to_qname(model_rel.linkQname.clarkNotation, dts)
     rel_info['arcrole'] = model_rel.arcrole
     rel_info['attributes'] = {resolve_clark_to_qname(k, dts):v for k, v in model_rel.arcElement.attrib.items()
-                               if k not in ('order', 'weight', 'preferredLabel', 'priority', 
+                               if k not in ('order', 'weight', 'preferredLabel', 'priority', 'use',
                                             '{http://www.w3.org/1999/xlink}arcrole',
                                             '{http://www.w3.org/1999/xlink}from',
                                             '{http://www.w3.org/1999/xlink}to',

diff --git a/plugin/xodel/xodel.py b/plugin/xodel/xodel.py
@@ -2910,7 +2910,7 @@ def document_sort(log_infos, cntlr):
         for log_info in docs[uri]:
             if 'document-namespace' in log_info[1].args:
                 if namespace is not None and namespace != log_info[1].args['document-namespace']:
-                    raise XodelException(f"Document {uri} has conficting namespaces: {namespace} and {log_info[1].args['document-namespacde']}")
+                    raise XodelException(f"Document {uri} has conficting namespaces: {namespace} and {log_info[1].args['document-namespace']}")
 
                 namespace = log_info[1].args['document-namespace']
                 namespace_recs.append(log_info)

diff --git a/plugin/xule/XuleModelIndexer.py b/plugin/xule/XuleModelIndexer.py
@@ -216,6 +216,11 @@ def index_property_balance(model_fact):
 # entityIdentifer[1] is the id
 }
 
+TABLE_INDEX_PROPERTIES = {
+    ('property', 'cube', 'name'),
+    ('property', 'cube', 'drs-role')
+}
+
 def index_table_properties(model):
     """"Add the table properites to the fact index
 

diff --git a/plugin/xule/XuleProcessor.py b/plugin/xule/XuleProcessor.py
@@ -325,25 +325,25 @@ def index_property_balance(model_fact):
 #    1 - 'property' - This part is always 'property'.
 #    2 - the aspect name
 #    3 - the property name
-_FACT_INDEX_PROPERTIES = {
-    ('property', 'concept', 'period-type'): lambda f: f.concept.periodType,
-    ('property', 'concept', 'balance'): index_property_balance,
-    ('property', 'concept', 'data-type'): lambda f: f.concept.typeQname,
-    ('property', 'concept', 'base-type'): lambda f: f.concept.baseXbrliTypeQname,
-    ('property', 'concept', 'is-monetary'): lambda f: f.concept.isMonetary,
-    ('property', 'concept', 'is-numeric'): lambda f: f.concept.isNumeric,
-    ('property', 'concept', 'substitution'): lambda f: f.concept.substitutionGroupQname,
-    ('property', 'concept', 'namespace-uri'): lambda f: f.concept.qname.namespaceURI,
-    ('property', 'concept', 'local-name'): lambda f: f.concept.qname.localName,
-    ('property', 'concept', 'is-abstract'): lambda f: f.concept.isAbstract,
-    ('property', 'concept', 'id'): lambda f: f.id,
-    ('property', 'period', 'start'): index_property_start,
-    ('property', 'period', 'end'): index_property_end,
-    ('property', 'period', 'days'): index_property_days,
-    ('property', 'entity', 'scheme'): lambda f: f.context.entityIdentifier[0],  # entityIdentifier[0] is the scheme
-    ('property', 'entity', 'id'): lambda f: f.context.entityIdentifier[1]
-# entityIdentifer[1] is the id
-}
+# _FACT_INDEX_PROPERTIES = {
+#     ('property', 'concept', 'period-type'): lambda f: f.concept.periodType,
+#     ('property', 'concept', 'balance'): index_property_balance,
+#     ('property', 'concept', 'data-type'): lambda f: f.concept.typeQname,
+#     ('property', 'concept', 'base-type'): lambda f: f.concept.baseXbrliTypeQname,
+#     ('property', 'concept', 'is-monetary'): lambda f: f.concept.isMonetary,
+#     ('property', 'concept', 'is-numeric'): lambda f: f.concept.isNumeric,
+#     ('property', 'concept', 'substitution'): lambda f: f.concept.substitutionGroupQname,
+#     ('property', 'concept', 'namespace-uri'): lambda f: f.concept.qname.namespaceURI,
+#     ('property', 'concept', 'local-name'): lambda f: f.concept.qname.localName,
+#     ('property', 'concept', 'is-abstract'): lambda f: f.concept.isAbstract,
+#     ('property', 'concept', 'id'): lambda f: f.id,
+#     ('property', 'period', 'start'): index_property_start,
+#     ('property', 'period', 'end'): index_property_end,
+#     ('property', 'period', 'days'): index_property_days,
+#     ('property', 'entity', 'scheme'): lambda f: f.context.entityIdentifier[0],  # entityIdentifier[0] is the scheme
+#     ('property', 'entity', 'id'): lambda f: f.context.entityIdentifier[1]
+# # entityIdentifer[1] is the id
+# }
 
 # def index_table_properties(xule_context):
 #     """"Add the table properites to the fact index
@@ -2638,7 +2638,7 @@ def fact_index_key(aspect_info, fact_index, xule_context):
         # aspect_info[ASPECT_PROPERTY][1] is a tuple of the arguments
         index_key = ('property', aspect_info[ASPECT], aspect_info[ASPECT_PROPERTY][0]) + \
                     aspect_info[ASPECT_PROPERTY][1]
-        if index_key not in fact_index and index_key not in _FACT_INDEX_PROPERTIES and aspect_info[ASPECT_PROPERTY][0] != 'attribute':
+        if index_key not in fact_index and index_key not in xmi.FACT_INDEX_PROPERTIES and index_key not in xmi.TABLE_INDEX_PROPERTIES and aspect_info[ASPECT_PROPERTY][0] != 'attribute':
             raise XuleProcessingError(_(
                 "Factset aspect property '{}' is not a valid property of aspect '{}'.".format(index_key[2],
                                                                                                 index_key[1])),

diff --git a/plugin/xule/version.json b/plugin/xule/version.json
@@ -1,4 +1,4 @@
 {
-  "version": "30041",
+  "version": "30043",
   "ruleset_version": "23752"
 }