Merge pull request #582 from VisLab/develop

Corrected the summary generation for single files

hed/tools/bids/bids_dataset.py

@@ -8,7 +8,7 @@
 from hed.validator.hed_validator import HedValidator
 
 
-LIBRARY_URL_BASE = "https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/library_schemas/"
+LIBRARY_URL_BASE = "https://raw.githubusercontent.com/hed-standard/hed-schemas/main/library_schemas/"
 
 
 class BidsDataset:
@@ -115,42 +115,3 @@ def get_schema_versions(self):
             name = prefix + name
             version_list.append(name)
         return version_list
-
-
-# if __name__ == '__main__':
-    # path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-    #                     '../../../tests/data/bids_tests/eeg_ds003654s_hed_library')
-    # path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-    #                     '../../../tests/data/bids_tests/eeg_ds003654s_hed_inheritance')
-    # path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-    #                     '../../../tests/data/bids_tests/eeg_ds003654s_hed')
-    #
-    # path = 'Q:\WakemanHensonON'
-    # path = 'G:\\WakemanHenson\\WH_Released'
-    # bids = BidsDataset(path)
-    # issue_list = bids.validate(check_for_warnings=False)
-    # if issue_list:
-    #     issue_str = get_printable_issue_string(issue_list, "HED validation errors:", skip_filename=False)
-    # else:
-    #     issue_str = "No issues"
-    # print(issue_str)
-    # warnings = False
-    # path = '/XXX/bids-examples/xeeg_hed_score/'
-    # bids = BidsDataset(path)
-    # # summary1 = bids.get_summary()
-    # # print(json.dumps(summary1, indent=4))
-    # print("\nNow validating with the prerelease schema.")
-    # base_version = '8.1.0'
-    # score_url = f"https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/library_schemas/" \
-    #             f"score/prerelease/HED_score_1.0.0.xml"
-    #
-    # schema_base = load_schema_version(xml_version="8.1.0")
-    # schema_score = load_schema(score_url, schema_prefix="sc")
-    # bids.schema = HedSchemaGroup([schema_base, schema_score])
-    #
-    # issue_list2 = bids.validate(check_for_warnings=warnings)
-    # if issue_list2:
-    #     issue_str2 = get_printable_issue_string(issue_list2, "HED validation errors: ", skip_filename=False)
-    # else:
-    #     issue_str2 = "No HED validation errors"
-    # print(issue_str2)

hed/tools/bids/bids_file.py

@@ -11,7 +11,6 @@ class BidsFile:
         ext (str):                   Extension (including the .).
         entity_dict (dict):          Dictionary of entity-names (keys) and entity-values (values).
         sidecar (BidsSidecarFile):   Merged sidecar for this file.
-        contents:                    Contents of this file
 
     Notes:
         - This class may hold the merged sidecar giving metadata for this file as well as contents.

hed/tools/bids/bids_file_group.py

@@ -215,35 +215,3 @@ def _make_sidecar_dir_dict(self):
                 new_dir_list.append(self.sidecar_dict[os.path.realpath(s_file)])
             sidecar_dir_dict[os.path.realpath(this_dir)] = new_dir_list
         return sidecar_dir_dict
-
-
-# if __name__ == '__main__':
-    # path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
-    #                     '../../../tests/data/bids_tests/eeg_ds003654s_hed_inheritance')
-    # path = 'G:\\WakemanHenson\\WH_Released'
-    # bids = BidsFileGroup(path)
-
-    # for file_obj in bids.sidecar_dict.values():
-    #     print(file_obj.file_path)
-    #
-    # for file_obj in bids.datafile_dict.values():
-    #     print(file_obj.file_path)
-
-    # schema = load_schema_version(xml_version="8.0.0")
-    # validator = HedValidator(hed_schema=schema)
-    #
-    # issues_side = bids.validate_sidecars(hed_ops=[validator], check_for_warnings=False)
-    # print(f"Side issues {str(issues_side)}\n")
-    # issues_data = bids.validate_datafiles(hed_ops=[validator], check_for_warnings=False)
-    # print(f"Data issues {str(issues_data)}\n")
-    # if issues_side:
-    #     print(get_printable_issue_string(issues_side, "Sidecar errors", skip_filename=False))
-    # else:
-    #     print("No validation errors in the sidecars")
-    #
-    # if issues_data:
-    #     print(get_printable_issue_string(issues_data, "Tabular errors"))
-    # else:
-    #     print("No validation errors in the datafiles")
-    # col_info = bids.get_summary()
-    # print(f"Info: {str(col_info)}")

hed/tools/bids/bids_sidecar_file.py

@@ -81,18 +81,3 @@ def is_hed(json_dict):
                 return True
 
         return False
-
-
-# if __name__ == '__main__':
-#     from hed import load_schema, HedValidator, load_schema_version, HedSchemaGroup
-#     score_url = f"https://raw.githubusercontent.com/hed-standard/hed-schema-library/main/library_schemas" \
-#                 f"/score/prerelease/HED_score_1.0.0.xml"
-#     path = f"../../../sub-eegArtifactTUH_ses-eeg01_task-rest_run-001_events.json"
-#     bids_json = BidsSidecarFile(path)
-#     bids_json.set_contents()
-#     schema_base = load_schema_version(xml_version="8.1.0")
-#     schema_score = load_schema(score_url, schema_prefix="sc")
-#     schemas = HedSchemaGroup([schema_base, schema_score])
-#     validator = HedValidator(hed_schema=schemas)
-#     issues = bids_json.contents.validate_entries(hed_ops=validator, check_for_warnings=False)
-#     print(f"issues:{str(issues)}")

hed/tools/remodeling/dispatcher.py

@@ -1,6 +1,7 @@
 import os
 import numpy as np
 import pandas as pd
+import json
 from hed.errors.exceptions import HedFileError
 from hed.schema.hed_schema_io import get_schema
 from hed.tools.remodeling.backup_manager import BackupManager
@@ -38,12 +39,11 @@ def __init__(self, operation_list, data_root=None,
         self.hed_schema = get_schema(hed_versions)
         self.context_dict = {}
 
-    def get_context_summaries(self, file_formats=['.txt', '.json'], individual_summaries="separated"):
-        """ Return the summaries in a dictionary suitable for saving or archiving.
+    def get_context_summaries(self, file_formats=['.txt', '.json']):
+        """ Return the summaries in a dictionary of strings suitable for saving or archiving.
 
         Parameters:
             file_formats (list):  List of formats for the context files ('.json' and '.txt' are allowed).
-            individual_summaries (str): Possible values are separate, consolidated, or none
 
         Returns:
             list: A list of dictionaries of summaries keyed to filenames.
@@ -55,9 +55,11 @@ def get_context_summaries(self, file_formats=['.txt', '.json'], individual_summa
             file_base = generate_filename(context_item.context_filename, append_datetime=True)
             for file_format in file_formats:
                 if file_format == '.txt':
-                    summary = context_item.get_text_summary(individual_summaries=individual_summaries)
+                    summary = context_item.get_text_summary(individual_summaries="consolidated")
+                    summary = summary['Dataset']
                 elif file_format == '.json':
-                    summary = context_item.get_summary(as_json=True)
+                    summary = json.dumps(context_item.get_summary(individual_summaries="consolidated"), indent=4)
+
                 else:
                     continue
                 summary_list.append({'file_name': file_base + file_format, 'file_format': file_format,

hed/tools/remodeling/operations/base_context.py

@@ -39,31 +39,28 @@ def get_summary_details(self, include_individual=True):
                 summary_details["Individual files"][name] = self._get_summary_details(count)
         return summary_details
 
-    def get_summary(self, as_json=True, individual_summaries="separate"):
+    def get_summary(self, individual_summaries="separate"):
+
         include_individual = individual_summaries == "separate" or individual_summaries == "consolidated"
         summary_details = self.get_summary_details(include_individual=include_individual)
-        summary = {"Dataset": {"Context name": self.context_name, "Context type": self.context_type,
-                   "Context filename": self.context_filename, "Overall summary": summary_details['Dataset']}}
-
-        if individual_summaries == "separate":
-            summary["Individual files"] = {}
-            for name, name_summary in summary_details["Individual files"].items():
-                summary["Individual files"][name] = {"Context name": self.context_name,
-                                                     "Context type": self.context_type,
-                                                     "Context filename": self.context_filename,
-                                                     "Summary": summary_details['Individual files'][name]}
-                if as_json:
-                    summary["Individual files"][name] = \
-                        json.dumps(summary_details["Individual files"][name], indent=4)
-            if as_json:
-                summary["Dataset"] = json.dumps(summary["Dataset"], indent=4)
-            return summary
+        dataset_summary = {"Context name": self.context_name, "Context type": self.context_type,
+                           "Context filename": self.context_filename, "Overall summary": summary_details['Dataset']}
+        summary = {"Dataset": dataset_summary, "Individual files": {}}
         if summary_details["Individual files"]:
-            summary["Dataset"]["Individual files"] = summary_details["Individual files"]
-        if as_json:
-            summary["Dataset"] = json.dumps(summary["Dataset"], indent=4)
+            summary["Individual files"] = self.get_individual(summary_details["Individual files"],
+                                                              separately=individual_summaries == "separate")
         return summary
 
+    def get_individual(self, summary_details, separately=True):
+        individual_dict = {}
+        for name, name_summary in summary_details.items():
+            if separately:
+                individual_dict[name] = {"Context name": self.context_name, "Context type": self.context_type,
+                                         "Context filename": self.context_filename, "File summary": name_summary}
+            else:
+                individual_dict[name] = name_summary
+        return individual_dict
+
     def get_text_summary_details(self, include_individual=True):
         result = self.get_summary_details(include_individual=include_individual)
         summary_details = {"Dataset": self._get_result_string("Dataset", result.get("Dataset", "")),
@@ -102,35 +99,37 @@ def save(self, save_dir, file_formats=['.txt'], individual_summaries="separate")
             if file_format == '.txt':
                 summary = self.get_text_summary(individual_summaries=individual_summaries)
             elif file_format == '.json':
-                summary = self.get_summary(as_json=True, individual_summaries=individual_summaries)
+                summary = self.get_summary(individual_summaries=individual_summaries)
             else:
                 continue
             self._save_separate(save_dir, file_format, time_stamp, summary)
 
     def _save_separate(self, save_dir, file_format, time_stamp, summary):
         this_save = os.path.join(save_dir, self.context_name + '/')
         os.makedirs(os.path.realpath(this_save), exist_ok=True)
-        file_name = os.path.realpath(os.path.join(this_save, secure_filename(self.context_filename) + "_overall" +
-                                                  time_stamp + file_format))
-
-        with open(file_name, 'w') as text_file:
-            text_file.write(summary["Dataset"])
+        filename = os.path.realpath(os.path.join(this_save, secure_filename(self.context_filename) + "_overall" +
+                                                 time_stamp + file_format))
+        self.dump_summary(filename, summary["Dataset"])
         individual = summary.get("Individual files", {})
-
         if not individual:
             return
-
         individual_dir = os.path.join(this_save, self.INDIVIDUAL_SUMMARIES_PATH + '/')
         os.makedirs(os.path.realpath(individual_dir), exist_ok=True)
         for name, sum_str in individual.items():
-            file_name = secure_filename(self.context_filename + "_" + name + time_stamp + file_format)
-            file_name = os.path.realpath(os.path.join(individual_dir, file_name))
-            with open(file_name, 'w') as text_file:
-                text_file.write(sum_str)
+            filename = secure_filename(self.context_filename + "_" + name + time_stamp + file_format)
+            filename = os.path.realpath(os.path.join(individual_dir, filename))
+            self.dump_summary(filename, sum_str)
 
     def _get_result_string(self, name, result, indent=DISPLAY_INDENT):
         return f"\n{name}\n{indent}{str(result)}"
 
+    @staticmethod
+    def dump_summary(filename, summary):
+        with open(filename, 'w') as text_file:
+            if not isinstance(summary, str):
+                summary = json.dumps(summary, indent=4)
+            text_file.write(summary)
+
     @abstractmethod
     def _get_summary_details(self, summary_info):
         """ Return the summary-specific information.

hed/tools/remodeling/operations/summarize_events_to_sidecar_op.py

@@ -79,7 +79,7 @@ def _get_summary_details(self, summary_info):
         """ Return the summary-specific information.
 
         Parameters:
-            summary_info (object):  Summary to return info from
+            summary_info (Object):  Summary to return info from
 
         Notes:
             Abstract method be implemented by each individual context summary.
@@ -102,9 +102,8 @@ def _merge_all(self):
         """
         return {}
 
-    def _get_result_string(self, name, result):
+    def _get_result_string(self, name, result, indent=DISPLAY_INDENT):
         if name == "Dataset":
             return "Dataset: Currently no overall sidecar extraction is available"
-        details = result['sidecar']
-        json_str = f"\nSidecar:\n{json.dumps(details['sidecar'], indent=4)}"
-        return f"{name}: Total events={details['total_events']} Skip columns: {str(details['skip_cols'])}{json_str}"
+        json_str = f"\nSidecar:\n{json.dumps(result['sidecar'], indent=4)}"
+        return f"{name}: Total events={result['total_events']} Skip columns: {str(result['skip_cols'])}{json_str}"

tests/data/bids_tests/eeg_ds003654s_hed_library/dataset_description.json

@@ -1,7 +1,7 @@
 {
     "Name": "Face processing MEEG dataset with HED annotation",
     "BIDSVersion": "1.8.4",
-    "HEDVersion": ["8.0.0", "sc:score_0.0.1",  "test:testlib_1.0.2"],
+    "HEDVersion": ["8.0.0", "sc:score_1.0.0",  "test:testlib_1.0.2"],
     "License": "CC0",
     "Authors": [
         "Daniel G. Wakeman",

tests/data/bids_tests/eeg_ds003654s_hed_library/task-FacePerception_events.json

@@ -25,7 +25,7 @@
       "show_face": "Sensory-event, test:Experimental-stimulus, (Def/Face-image, Onset), (Def/Blink-inhibition-task,Onset),(Def/Cross-only, Offset)",
       "show_face_initial": "Sensory-event, Experimental-stimulus, (Def/Face-image, Onset), (Def/Blink-inhibition-task,Onset), (Def/Fixation-task, Onset)",
       "show_circle": "Sensory-event, (Intended-effect, Cue), (Def/Circle-only, Onset), (Def/Face-image, Offset), (Def/Blink-inhibition-task, Offset), (Def/Fixation-task, Offset)",
-      "show_cross": "Sensory-event, (Intended-effect, Cue), (Def/Cross-only, Onset), (Def/Fixation-task, Onset),  (Def/Circle-only, Offset), sc:Quality-of-hyperventilation",
+      "show_cross": "Sensory-event, (Intended-effect, Cue), (Def/Cross-only, Onset), (Def/Fixation-task, Onset),  (Def/Circle-only, Offset), sc:Hyperventilation-refused-procedure/Could not comply",
       "left_press": "Agent-action, Participant-response, Def/Press-left-finger",
       "right_press": "Agent-action, Participant-response, Def/Press-right-finger",
       "setup_left_sym": "Experiment-structure, (Def/Left-sym-cond, Onset), (Def/Initialize-recording, Onset)",

tests/tools/remodeling/operations/test_base_context.py

@@ -76,14 +76,14 @@ def test_save(self):
         self.assertFalse(os.path.isdir(dir_ind))
         test1.save(self.summary_dir, file_formats=['.json', '.tsv'], individual_summaries="consolidated")
         file_list3 = os.listdir(dir_full)
-        self.assertEqual(len(file_list3), len(file_list2) + 1)
-        self.assertFalse(os.path.isdir(dir_ind))
+        self.assertEqual(len(file_list3), len(file_list2) + 2)
+        self.assertTrue(os.path.isdir(dir_ind))
         test1.save(self.summary_dir, file_formats=['.json', '.tsv'], individual_summaries="separate")
         self.assertTrue(os.path.isdir(dir_ind))
         file_list4 = os.listdir(dir_full)
-        self.assertEqual(len(file_list4), len(file_list3) + 2)
+        self.assertEqual(len(file_list4), len(file_list3) + 1)
         file_list5 = os.listdir(dir_ind)
-        self.assertEqual(len(file_list5), 2)
+        self.assertEqual(len(file_list5), 4)
 
 
 if __name__ == '__main__':

tests/tools/remodeling/operations/test_summarize_column_names_op.py

@@ -96,8 +96,8 @@ def test_summary(self):
         context = dispatch.context_dict['columns']
         summary = context.get_summary()
         dataset_sum = summary['Dataset']
-        json_obj = json.loads(dataset_sum)
-        json_str = json.dumps(json_obj)
+        json_str = json.dumps(dataset_sum)
+        json_obj = json.loads(json_str)
         columns = json_obj["Overall summary"]["Columns"]
         self.assertEqual(len(columns), 1)
         self.assertEqual(len(columns[0]['Files']), 2)

tests/tools/remodeling/operations/test_summarize_column_values_op.py

@@ -70,12 +70,12 @@ def test_get_summary(self):
         cont = dispatch.context_dict
         context1 = cont.get("test summary", None)
         self.assertIsInstance(context1, ColumnValueSummaryContext, "get_summary testing ColumnValueSummary")
-        summary1 = context1.get_summary(as_json=False)
+        summary1 = context1.get_summary()
         self.assertIsInstance(summary1, dict, "get_summary returns a dictionary")
         self.assertIsInstance(summary1["Dataset"], dict)
-        summary1a = context1.get_summary(as_json=True)
+        summary1a = context1.get_summary()
         self.assertIsInstance(summary1a, dict)
-        self.assertIsInstance(summary1a["Dataset"], str)
+        self.assertIsInstance(summary1a["Dataset"], dict)
         text_summary = context1.get_text_summary(individual_summaries="separate")
         self.assertIsInstance(text_summary, dict)
         self.assertIsInstance(text_summary["Dataset"], str)
@@ -105,7 +105,7 @@ def test_summary_op(self):
         for key, item in context_dict.items():
             text_value = item.get_text_summary()
             self.assertTrue(text_value)
-            json_value = item.get_summary(as_json=True)
+            json_value = item.get_summary()
             self.assertTrue(json_value)