From e9e63e39f6a99487c5e69ae96e653c4580045ec0 Mon Sep 17 00:00:00 2001 From: Kay Robbins <1189050+VisLab@users.noreply.github.com> Date: Thu, 12 Oct 2023 10:29:43 -0500 Subject: [PATCH] Corrected an error in the formatting of text output of validation summary --- hed/tools/remodeling/cli/run_remodel.py | 18 +++++++++-------- .../operations/summarize_hed_tags_op.py | 2 ++ .../operations/summarize_hed_validation_op.py | 20 +++++++++++-------- tests/models/test_hed_group.py | 2 +- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/hed/tools/remodeling/cli/run_remodel.py b/hed/tools/remodeling/cli/run_remodel.py index 45fce71a..8aefdb15 100644 --- a/hed/tools/remodeling/cli/run_remodel.py +++ b/hed/tools/remodeling/cli/run_remodel.py @@ -142,21 +142,21 @@ def run_bids_ops(dispatch, args, tabular_files): dispatch.hed_schema = bids.schema if args.verbose: print(f"Successfully parsed BIDS dataset with HED schema {str(bids.schema.get_schema_versions())}") - events = bids.get_tabular_group(args.file_suffix) + data = bids.get_tabular_group(args.file_suffix) if args.verbose: print(f"Processing {dispatch.data_root}") - filtered_events = [events.datafile_dict[key] for key in tabular_files] - for events_obj in filtered_events: - sidecar_list = events.get_sidecars_from_path(events_obj) + filtered_events = [data.datafile_dict[key] for key in tabular_files] + for data_obj in filtered_events: + sidecar_list = data.get_sidecars_from_path(data_obj) if sidecar_list: - sidecar = events.sidecar_dict[sidecar_list[-1]].contents + sidecar = data.sidecar_dict[sidecar_list[-1]].contents else: sidecar = None if args.verbose: - print(f"Events {events_obj.file_path} sidecar {sidecar}") - df = dispatch.run_operations(events_obj.file_path, sidecar=sidecar, verbose=args.verbose) + print(f"Tabular file {data_obj.file_path} sidecar {sidecar}") + df = dispatch.run_operations(data_obj.file_path, sidecar=sidecar, verbose=args.verbose) if not args.no_update: - df.to_csv(events_obj.file_path, sep='\t', index=False, header=True) + df.to_csv(data_obj.file_path, sep='\t', index=False, header=True) def run_direct_ops(dispatch, args, tabular_files): @@ -176,6 +176,8 @@ def run_direct_ops(dispatch, args, tabular_files): else: sidecar = None for file_path in tabular_files: + if args.verbose: + print(f"Tabular file {file_path} sidecar {sidecar}") df = dispatch.run_operations(file_path, verbose=args.verbose, sidecar=sidecar) if not args.no_update: df.to_csv(file_path, sep='\t', index=False, header=True) diff --git a/hed/tools/remodeling/operations/summarize_hed_tags_op.py b/hed/tools/remodeling/operations/summarize_hed_tags_op.py index ffef53fb..6e98abbe 100644 --- a/hed/tools/remodeling/operations/summarize_hed_tags_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_tags_op.py @@ -87,6 +87,8 @@ def do_op(self, dispatcher, df, name, sidecar=None): if not summary: summary = HedTagSummary(self) dispatcher.summary_dicts[self.summary_name] = summary + x = {'df': dispatcher.post_proc_data(df_new), 'name': name, + 'schema': dispatcher.hed_schema, 'sidecar': sidecar} summary.update_summary({'df': dispatcher.post_proc_data(df_new), 'name': name, 'schema': dispatcher.hed_schema, 'sidecar': sidecar}) return df_new diff --git a/hed/tools/remodeling/operations/summarize_hed_validation_op.py b/hed/tools/remodeling/operations/summarize_hed_validation_op.py index cd3fc936..a2948eb8 100644 --- a/hed/tools/remodeling/operations/summarize_hed_validation_op.py +++ b/hed/tools/remodeling/operations/summarize_hed_validation_op.py @@ -1,7 +1,7 @@ """ Validate the HED tags in a dataset and report errors. """ import os -from hed.errors import ErrorSeverity, ErrorHandler +from hed.errors import ErrorSeverity, ErrorHandler, get_printable_issue_string from hed.models.sidecar import Sidecar from hed.models.tabular_input import TabularInput from hed.tools.remodeling.operations.base_op import BaseOp @@ -110,9 +110,9 @@ def _get_result_string(self, name, result, indent=BaseSummary.DISPLAY_INDENT): else: sum_list = sum_list + self.get_error_list(specifics['sidecar_issues'], indent=indent*2) if specifics['sidecar_had_issues']: - sum_list = sum_list + self.get_error_list(specifics['event_issues'], count_only=False, indent=indent*2) + sum_list = sum_list + self.get_error_list(specifics['sidecar_issues'], count_only=False, indent=indent*2) else: - sum_list = sum_list + [f"{indent*2}Event file validation was incomplete because of sidecar errors"] + sum_list = sum_list + self.get_error_list(specifics['event_issues'], count_only=False, indent=indent*2) return "\n".join(sum_list) def update_summary(self, new_info): @@ -134,6 +134,7 @@ def update_summary(self, new_info): issues = input_data.validate(new_info['schema']) if not self.check_for_warnings: issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR) + issues = [get_printable_issue_string([issue], skip_filename=True) for issue in issues] results['event_issues'][new_info["name"]] = issues results['total_event_issues'] = len(issues) self.summary_dict[new_info["name"]] = results @@ -197,13 +198,15 @@ def get_error_list(error_dict, count_only=False, indent=BaseSummary.DISPLAY_INDE error_list = [] for key, item in error_dict.items(): if count_only and isinstance(item, list): - error_list.append(f"{indent}{key}: {len(item)} issues") + error_list.append(f"{key}: {len(item)} issues") elif count_only: - error_list.append(f"{indent}{key}: {item} issues") + error_list.append(f"{key}: {item} issues") elif not len(item): - error_list.append(f"{indent}{key} has no issues") + error_list.append(f"{key} has no issues") else: - HedValidationSummary._format_errors(error_list, key, item, indent) + error_list.append(f"{key}:") + error_list = error_list + item + #HedValidationSummary._format_errors(error_list, key, item, indent) return error_list @staticmethod @@ -246,6 +249,7 @@ def _get_sidecar_results(sidecar, new_info, check_for_warnings): results["sidecar_had_issues"] = True if not check_for_warnings: sidecar_issues = filtered_issues - results['sidecar_issues'][sidecar.name] = sidecar_issues + str_issues = [get_printable_issue_string([issue], skip_filename=True) for issue in sidecar_issues] + results['sidecar_issues'][sidecar.name] = str_issues results['total_sidecar_issues'] = len(sidecar_issues) return results diff --git a/tests/models/test_hed_group.py b/tests/models/test_hed_group.py index 117872e8..1a3632bf 100644 --- a/tests/models/test_hed_group.py +++ b/tests/models/test_hed_group.py @@ -62,7 +62,7 @@ def test_find_tags_with_term(self): # located tags now has found all 5 hed tags # This will find no tags - located_tags = basic_hed_string_obj.find_tags_with_term("bject", recursive=True, include_groups=0) + located_tags = basic_hed_string_obj.find_tags_with_term("reject", recursive=True, include_groups=0) self.assertEqual(len(located_tags), 0) # this will also find no tags