From a2499e04f800c444e913baa63c95423bfcff8158 Mon Sep 17 00:00:00 2001 From: David Tauriello Date: Wed, 11 Dec 2024 12:48:03 -0500 Subject: [PATCH] Update for 30043 - add handling to semanticHash for when concepts and other parameters don't exist in filing. (typically ESEF) - 0568591 - fix situational None return in semanticHash - 4b9b951 - refactor XuleValue format_value method to correctly include line number start rules - Xendr fix line numbers (#78) missing fact (#79) - refactor XuleModelIndexer.py and XuleProcessor.py to list the cube properties in the model index for factsets. - refactor TABLE_INDEX_PROPERTIES to be a set instead of a dictionary. - Xodel fix attribute filtering in extract_rel_info and correct typo in document_sort exception message (#81) --- plugin/semanticHash.py | 32 +++++++++++++++++++++----- plugin/xendr/xendrRun.py | 11 +++++++-- plugin/xodel/arelleHelper.py | 2 +- plugin/xodel/xodel.py | 2 +- plugin/xule/XuleModelIndexer.py | 5 +++++ plugin/xule/XuleProcessor.py | 40 ++++++++++++++++----------------- plugin/xule/version.json | 2 +- 7 files changed, 64 insertions(+), 30 deletions(-) diff --git a/plugin/semanticHash.py b/plugin/semanticHash.py index dabecef5d..409276520 100644 --- a/plugin/semanticHash.py +++ b/plugin/semanticHash.py @@ -47,6 +47,7 @@ def semanticHashOptions(parser): parserGroup.add_option("--semantic-hash-string", action="store", help=_("File name to save the canonicalized string")) + def semanticHashUtilityRun(cntlr, options, *arg, **kwargs): # save the controller and the options for use later. @@ -244,19 +245,26 @@ def semanticHashFactValue(fact): original_text_block_qname = qname('http://www.xbrl.org/dtr/type/non-numeric', 'textBlockItemType') new_text_block_qname = qname('http://www.xbrl.org/dtr/type/2020-01-21', 'textBlockItemType') fraction_qname = qname('http://www.xbrl.org/2003/instance', 'fractionItemType') - if fact.concept.instanceOfType(original_text_block_qname) or fact.concept.instanceOfType(new_text_block_qname): + + # Unable to identify the concept (fact.concept is likely None) + if fact.concept is None: + base_type = 'string' + elif fact.concept.instanceOfType(original_text_block_qname) or fact.concept.instanceOfType(new_text_block_qname): base_type = 'textBlock' elif fact.concept.instanceOfType(fraction_qname): base_type = 'fraction' else: base_type = fact.concept.baseXsdType + if base_type == 'QName': string_value = (fact.xValue.namespaceURI, fact.xValue.localName) elif base_type == 'textBlock': string_value = fact elif base_type == 'fraction': string_value = fact + elif base_type == 'string': + string_value = str(fact.value) else: string_value = fact.value @@ -281,12 +289,21 @@ def canonicalizeValue(base_type, string_value): elif base_type == 'fraction': return_value = value_string_hasher(string_value) else: - # This re will collapse whitespaces - return_value = value_string_hasher(re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' ')) + try: + # This re will collapse whitespaces + return_value = value_string_hasher(re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' ')) + except SemanticHashException: + return_value = re.sub(r"[ \t\n\r]+", ' ', string_value).strip(' ') + + + return return_value def semanticFormat(name, str_val): + if str(type(str_val)) != "": + str_val = '' + return "{}{}.{}".format(name.upper(), len(str_val), str_val) def UOMDefault(unit): @@ -301,7 +318,7 @@ def UOMDefault(unit): def canonicalizeTypedDimensionMember(member): element = member.modelXbrl.qnameConcepts.get(member.elementQname) - base_type = element.baseXsdType + base_type = element.baseXsdType if element is not None else 'string' if base_type == 'QName': string_value = (member.xValue.namespaceURI, member.xValue.localName) else: @@ -437,7 +454,12 @@ def canonicalizeTextBlock(content): # The wrapper allows fragments that don't start with a tag to be parsed. i.e. "abc
this is in the div
". This # example would not parse becasue it starts with text and not a tag. wrapper = '{}'.format(content.value.strip()) - xml_content = etree.fromstring(wrapper) + try: + p = etree.XMLParser(recover=True) + xml_content = etree.XML(wrapper, parser=p) + except Exception as ex: + raise SemanticHashException("XMLError", + _("The error is: " + str(ex) + "||" + wrapper)) return_value = canonicalizeXML(xml_content, include_node_tag=False) return return_value diff --git a/plugin/xendr/xendrRun.py b/plugin/xendr/xendrRun.py index 78ff13a46..e829405b9 100644 --- a/plugin/xendr/xendrRun.py +++ b/plugin/xendr/xendrRun.py @@ -285,7 +285,7 @@ def substitute_rule(rule_name, sub_info, line_number_subs, rule_results, templat # parent_classes = [] if is_actual_fact(json_result, modelXbrl): # get modelFact - if json_result['fact'] is not None: + if json_result.get('fact') is not None: model_fact = get_model_object(json_result['fact'], cntlr) @@ -1277,7 +1277,14 @@ def render_report(cntlr, options, modelXbrl, *args, **kwargs): rule_meta_data = {'substitutions': meta_substitutions, 'line-numbers': meta_line_number_subs} - rule_results = run_xule_rules(cntlr, options, modelXbrl, ts, rule_meta_data['standard-rules'] + list(rule_meta_data['showifs'].keys()), catalog_item['xule-rule-set']) + # Gather line number start rules + line_number_rules = set() + for line_number_info in rule_meta_data['line-numbers'].values(): + for line_number_item in line_number_info: + if 'start-rule' in line_number_item: + line_number_rules.add(line_number_item['start-rule']) + + rule_results = run_xule_rules(cntlr, options, modelXbrl, ts, rule_meta_data['standard-rules'] + list(rule_meta_data['showifs'].keys()) + list(line_number_rules), catalog_item['xule-rule-set']) # Substitute template template_number += 1 diff --git a/plugin/xodel/arelleHelper.py b/plugin/xodel/arelleHelper.py index ddd705297..462344ea9 100644 --- a/plugin/xodel/arelleHelper.py +++ b/plugin/xodel/arelleHelper.py @@ -303,7 +303,7 @@ def extract_rel_info(model_rel, dts): rel_info['type'] = resolve_clark_to_qname(model_rel.linkQname.clarkNotation, dts) rel_info['arcrole'] = model_rel.arcrole rel_info['attributes'] = {resolve_clark_to_qname(k, dts):v for k, v in model_rel.arcElement.attrib.items() - if k not in ('order', 'weight', 'preferredLabel', 'priority', + if k not in ('order', 'weight', 'preferredLabel', 'priority', 'use', '{http://www.w3.org/1999/xlink}arcrole', '{http://www.w3.org/1999/xlink}from', '{http://www.w3.org/1999/xlink}to', diff --git a/plugin/xodel/xodel.py b/plugin/xodel/xodel.py index dcaaf2f54..d50e437c5 100644 --- a/plugin/xodel/xodel.py +++ b/plugin/xodel/xodel.py @@ -2910,7 +2910,7 @@ def document_sort(log_infos, cntlr): for log_info in docs[uri]: if 'document-namespace' in log_info[1].args: if namespace is not None and namespace != log_info[1].args['document-namespace']: - raise XodelException(f"Document {uri} has conficting namespaces: {namespace} and {log_info[1].args['document-namespacde']}") + raise XodelException(f"Document {uri} has conficting namespaces: {namespace} and {log_info[1].args['document-namespace']}") namespace = log_info[1].args['document-namespace'] namespace_recs.append(log_info) diff --git a/plugin/xule/XuleModelIndexer.py b/plugin/xule/XuleModelIndexer.py index 52f0d5555..eb31d18b1 100644 --- a/plugin/xule/XuleModelIndexer.py +++ b/plugin/xule/XuleModelIndexer.py @@ -216,6 +216,11 @@ def index_property_balance(model_fact): # entityIdentifer[1] is the id } +TABLE_INDEX_PROPERTIES = { + ('property', 'cube', 'name'), + ('property', 'cube', 'drs-role') +} + def index_table_properties(model): """"Add the table properites to the fact index diff --git a/plugin/xule/XuleProcessor.py b/plugin/xule/XuleProcessor.py index f07c9d1fa..35c17bfd1 100644 --- a/plugin/xule/XuleProcessor.py +++ b/plugin/xule/XuleProcessor.py @@ -325,25 +325,25 @@ def index_property_balance(model_fact): # 1 - 'property' - This part is always 'property'. # 2 - the aspect name # 3 - the property name -_FACT_INDEX_PROPERTIES = { - ('property', 'concept', 'period-type'): lambda f: f.concept.periodType, - ('property', 'concept', 'balance'): index_property_balance, - ('property', 'concept', 'data-type'): lambda f: f.concept.typeQname, - ('property', 'concept', 'base-type'): lambda f: f.concept.baseXbrliTypeQname, - ('property', 'concept', 'is-monetary'): lambda f: f.concept.isMonetary, - ('property', 'concept', 'is-numeric'): lambda f: f.concept.isNumeric, - ('property', 'concept', 'substitution'): lambda f: f.concept.substitutionGroupQname, - ('property', 'concept', 'namespace-uri'): lambda f: f.concept.qname.namespaceURI, - ('property', 'concept', 'local-name'): lambda f: f.concept.qname.localName, - ('property', 'concept', 'is-abstract'): lambda f: f.concept.isAbstract, - ('property', 'concept', 'id'): lambda f: f.id, - ('property', 'period', 'start'): index_property_start, - ('property', 'period', 'end'): index_property_end, - ('property', 'period', 'days'): index_property_days, - ('property', 'entity', 'scheme'): lambda f: f.context.entityIdentifier[0], # entityIdentifier[0] is the scheme - ('property', 'entity', 'id'): lambda f: f.context.entityIdentifier[1] -# entityIdentifer[1] is the id -} +# _FACT_INDEX_PROPERTIES = { +# ('property', 'concept', 'period-type'): lambda f: f.concept.periodType, +# ('property', 'concept', 'balance'): index_property_balance, +# ('property', 'concept', 'data-type'): lambda f: f.concept.typeQname, +# ('property', 'concept', 'base-type'): lambda f: f.concept.baseXbrliTypeQname, +# ('property', 'concept', 'is-monetary'): lambda f: f.concept.isMonetary, +# ('property', 'concept', 'is-numeric'): lambda f: f.concept.isNumeric, +# ('property', 'concept', 'substitution'): lambda f: f.concept.substitutionGroupQname, +# ('property', 'concept', 'namespace-uri'): lambda f: f.concept.qname.namespaceURI, +# ('property', 'concept', 'local-name'): lambda f: f.concept.qname.localName, +# ('property', 'concept', 'is-abstract'): lambda f: f.concept.isAbstract, +# ('property', 'concept', 'id'): lambda f: f.id, +# ('property', 'period', 'start'): index_property_start, +# ('property', 'period', 'end'): index_property_end, +# ('property', 'period', 'days'): index_property_days, +# ('property', 'entity', 'scheme'): lambda f: f.context.entityIdentifier[0], # entityIdentifier[0] is the scheme +# ('property', 'entity', 'id'): lambda f: f.context.entityIdentifier[1] +# # entityIdentifer[1] is the id +# } # def index_table_properties(xule_context): # """"Add the table properites to the fact index @@ -2638,7 +2638,7 @@ def fact_index_key(aspect_info, fact_index, xule_context): # aspect_info[ASPECT_PROPERTY][1] is a tuple of the arguments index_key = ('property', aspect_info[ASPECT], aspect_info[ASPECT_PROPERTY][0]) + \ aspect_info[ASPECT_PROPERTY][1] - if index_key not in fact_index and index_key not in _FACT_INDEX_PROPERTIES and aspect_info[ASPECT_PROPERTY][0] != 'attribute': + if index_key not in fact_index and index_key not in xmi.FACT_INDEX_PROPERTIES and index_key not in xmi.TABLE_INDEX_PROPERTIES and aspect_info[ASPECT_PROPERTY][0] != 'attribute': raise XuleProcessingError(_( "Factset aspect property '{}' is not a valid property of aspect '{}'.".format(index_key[2], index_key[1])), diff --git a/plugin/xule/version.json b/plugin/xule/version.json index e339664ca..fbec198b2 100644 --- a/plugin/xule/version.json +++ b/plugin/xule/version.json @@ -1,4 +1,4 @@ { - "version": "30041", + "version": "30043", "ruleset_version": "23752" } \ No newline at end of file