Merge pull request #776 from VisLab/develop

Corrected a format error in txt output of validation summary

hed/tools/remodeling/cli/run_remodel.py

@@ -142,21 +142,21 @@ def run_bids_ops(dispatch, args, tabular_files):
     dispatch.hed_schema = bids.schema
     if args.verbose:
         print(f"Successfully parsed BIDS dataset with HED schema {str(bids.schema.get_schema_versions())}")
-    events = bids.get_tabular_group(args.file_suffix)
+    data = bids.get_tabular_group(args.file_suffix)
     if args.verbose:
         print(f"Processing {dispatch.data_root}")
-    filtered_events = [events.datafile_dict[key] for key in tabular_files]
-    for events_obj in filtered_events:
-        sidecar_list = events.get_sidecars_from_path(events_obj)
+    filtered_events = [data.datafile_dict[key] for key in tabular_files]
+    for data_obj in filtered_events:
+        sidecar_list = data.get_sidecars_from_path(data_obj)
         if sidecar_list:
-            sidecar = events.sidecar_dict[sidecar_list[-1]].contents
+            sidecar = data.sidecar_dict[sidecar_list[-1]].contents
         else:
             sidecar = None
         if args.verbose:
-            print(f"Events {events_obj.file_path}  sidecar {sidecar}")
-        df = dispatch.run_operations(events_obj.file_path, sidecar=sidecar, verbose=args.verbose)
+            print(f"Tabular file {data_obj.file_path}  sidecar {sidecar}")
+        df = dispatch.run_operations(data_obj.file_path, sidecar=sidecar, verbose=args.verbose)
         if not args.no_update:
-            df.to_csv(events_obj.file_path, sep='\t', index=False, header=True)
+            df.to_csv(data_obj.file_path, sep='\t', index=False, header=True)
 
 
 def run_direct_ops(dispatch, args, tabular_files):
@@ -176,6 +176,8 @@ def run_direct_ops(dispatch, args, tabular_files):
     else:
         sidecar = None
     for file_path in tabular_files:
+        if args.verbose:
+            print(f"Tabular file {file_path}  sidecar {sidecar}")
         df = dispatch.run_operations(file_path, verbose=args.verbose, sidecar=sidecar)
         if not args.no_update:
             df.to_csv(file_path, sep='\t', index=False, header=True)

hed/tools/remodeling/operations/summarize_hed_tags_op.py

@@ -87,6 +87,8 @@ def do_op(self, dispatcher, df, name, sidecar=None):
         if not summary:
             summary = HedTagSummary(self)
             dispatcher.summary_dicts[self.summary_name] = summary
+        x = {'df': dispatcher.post_proc_data(df_new), 'name': name,
+                                'schema': dispatcher.hed_schema, 'sidecar': sidecar}
         summary.update_summary({'df': dispatcher.post_proc_data(df_new), 'name': name,
                                 'schema': dispatcher.hed_schema, 'sidecar': sidecar})
         return df_new

hed/tools/remodeling/operations/summarize_hed_validation_op.py

@@ -1,7 +1,7 @@
 """ Validate the HED tags in a dataset and report errors. """
 
 import os
-from hed.errors import ErrorSeverity, ErrorHandler
+from hed.errors import ErrorSeverity, ErrorHandler, get_printable_issue_string
 from hed.models.sidecar import Sidecar
 from hed.models.tabular_input import TabularInput
 from hed.tools.remodeling.operations.base_op import BaseOp
@@ -110,9 +110,9 @@ def _get_result_string(self, name, result, indent=BaseSummary.DISPLAY_INDENT):
         else:
             sum_list = sum_list + self.get_error_list(specifics['sidecar_issues'], indent=indent*2)
             if specifics['sidecar_had_issues']:
-                sum_list = sum_list + self.get_error_list(specifics['event_issues'], count_only=False, indent=indent*2)
+                sum_list = sum_list + self.get_error_list(specifics['sidecar_issues'], count_only=False, indent=indent*2)
             else:
-                sum_list = sum_list + [f"{indent*2}Event file validation was incomplete because of sidecar errors"]
+                sum_list = sum_list + self.get_error_list(specifics['event_issues'], count_only=False, indent=indent*2)
         return "\n".join(sum_list)
 
     def update_summary(self, new_info):
@@ -134,6 +134,7 @@ def update_summary(self, new_info):
             issues = input_data.validate(new_info['schema'])
             if not self.check_for_warnings:
                 issues = ErrorHandler.filter_issues_by_severity(issues, ErrorSeverity.ERROR)
+            issues = [get_printable_issue_string([issue], skip_filename=True) for issue in issues]
             results['event_issues'][new_info["name"]] = issues
             results['total_event_issues'] = len(issues)
         self.summary_dict[new_info["name"]] = results
@@ -197,13 +198,15 @@ def get_error_list(error_dict, count_only=False, indent=BaseSummary.DISPLAY_INDE
         error_list = []
         for key, item in error_dict.items():
             if count_only and isinstance(item, list):
-                error_list.append(f"{indent}{key}: {len(item)} issues")
+                error_list.append(f"{key}: {len(item)} issues")
             elif count_only:
-                error_list.append(f"{indent}{key}: {item} issues")
+                error_list.append(f"{key}: {item} issues")
             elif not len(item):
-                error_list.append(f"{indent}{key} has no issues")
+                error_list.append(f"{key} has no issues")
             else:
-                HedValidationSummary._format_errors(error_list, key, item, indent)
+                error_list.append(f"{key}:")
+                error_list = error_list + item
+                #HedValidationSummary._format_errors(error_list, key, item, indent)
         return error_list
 
     @staticmethod
@@ -246,6 +249,7 @@ def _get_sidecar_results(sidecar, new_info, check_for_warnings):
                 results["sidecar_had_issues"] = True
             if not check_for_warnings:
                 sidecar_issues = filtered_issues
-            results['sidecar_issues'][sidecar.name] = sidecar_issues
+            str_issues = [get_printable_issue_string([issue], skip_filename=True) for issue in sidecar_issues]
+            results['sidecar_issues'][sidecar.name] = str_issues
             results['total_sidecar_issues'] = len(sidecar_issues)
         return results

tests/models/test_hed_group.py

@@ -62,7 +62,7 @@ def test_find_tags_with_term(self):
         # located tags now has found all 5 hed tags
 
         # This will find no tags
-        located_tags = basic_hed_string_obj.find_tags_with_term("bject", recursive=True, include_groups=0)
+        located_tags = basic_hed_string_obj.find_tags_with_term("reject", recursive=True, include_groups=0)
         self.assertEqual(len(located_tags), 0)
 
         # this will also find no tags