Merge pull request #602 from VisLab/develop

Updated the name of 3645 to correct for digit reversal

hed/tools/analysis/hed_tag_counts.py

@@ -4,19 +4,19 @@
 
 
 class HedTagCount:
-    def __init__(self, hed_tag, name):
+    def __init__(self, hed_tag, file_name):
         """ Keeps the counts for a particular HedTag.
 
         Parameters:
             hed_tag (HedTag):  The HedTag to keep track of.
-            name (str):   Name of the file associated with the tag.
+            file_name (str):   Name of the file associated with the tag.
 
         """
 
         self.tag = hed_tag.short_base_tag
         self.tag_terms = hed_tag.tag_terms
         self.events = 1
-        self.files = {name: ''}
+        self.files = {file_name: ''}
         self.value_dict = {}
         self.set_value(hed_tag)
 
@@ -37,6 +37,9 @@ def get_info(self, verbose=False):
         else:
             files = len(self.files)
         return {'tag': self.tag, 'events': self.events, 'files': files}
+
+    def get_summary(self):
+        return {'tag': self.tag, 'events': self.events, 'files': [name for name in self.files]}
 
     def get_empty(self):
         empty = copy.copy(self)
@@ -47,26 +50,30 @@ def get_empty(self):
 
 
 class HedTagCounts:
-    """ Keeps a summary of tag counts for a file.
+    """ Keeps a summary of tag counts for a tabular file.
 
 
     """
 
-    def __init__(self, name=""):
+    def __init__(self, name, total_events=0):
         self.tag_dict = {}
         self.name = name
-
-    def update_event_counts(self, hed_string_obj):
+        self.files = {}
+        self.total_events = total_events
+
+    def update_event_counts(self, hed_string_obj, file_name):
+        if file_name not in self.files:
+            self.files[file_name] = ""
         tag_list = hed_string_obj.get_all_tags()
         tag_dict = {}
         for tag in tag_list:
             str_tag = tag.short_base_tag.lower()
             if str_tag not in tag_dict:
-                tag_dict[str_tag] = HedTagCount(tag, self.name)
+                tag_dict[str_tag] = HedTagCount(tag, file_name)
             else:
                 tag_dict[str_tag].set_value(tag)
 
-        self.merge_tag_dicts(self.tag_dict, tag_dict)
+        self.merge_tag_dicts(tag_dict)
 
     def organize_tags(self, tag_template):
         template = self.create_template(tag_template)
@@ -82,25 +89,31 @@ def organize_tags(self, tag_template):
                 unmatched.append(tag_count)
         return template, unmatched
 
-    @staticmethod
-    def create_template(tags):
-        template_dict = {}
-        for key, key_list in tags.items():
-            for element in key_list:
-                template_dict[element.lower()] = []
-        return template_dict
-
-    @staticmethod
-    def merge_tag_dicts(base_dict, other_dict):
+    def merge_tag_dicts(self, other_dict):
         for tag, count in other_dict.items():
-            if tag not in base_dict:
-                base_dict[tag] = count.get_empty()
-            base_dict[tag].events = base_dict[tag].events + count.events
-            value_dict = base_dict[tag].value_dict
+            if tag not in self.tag_dict:
+                self.tag_dict[tag] = count.get_empty()
+            self.tag_dict[tag].events = self.tag_dict[tag].events + count.events
+            value_dict = self.tag_dict[tag].value_dict
             for value, val_count in count.value_dict.items():
                 if value in value_dict:
                     value_dict[value] = value_dict[value] + val_count
                 else:
                     value_dict[value] = val_count
             for file in count.files:
-                base_dict[tag].files[file] = ''
+                self.tag_dict[tag].files[file] = ''
+
+    def get_summary(self):
+        details = {}
+        for tag, count in self.tag_dict.items():
+            details[tag] = count.get_summary()
+        return {'name': str(self.name), 'type_tag': self.type_tag, 'files': list(self.files.keys()),
+                'total_events': self.total_events, 'details': details}
+
+    @staticmethod
+    def create_template(tags):
+        template_dict = {}
+        for key, key_list in tags.items():
+            for element in key_list:
+                template_dict[element.lower()] = []
+        return template_dict

hed/tools/analysis/hed_type_manager.py

@@ -109,7 +109,7 @@ def __str__(self):
 #     schema = load_schema_version(xml_version="8.1.0")
 #
 #     bids_root_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
-#                                                    '../../../tests/data/bids_tests/eeg_ds003654s_hed'))
+#                                                    '../../../tests/data/bids_tests/eeg_ds003645s_hed'))
 #     events_path = os.path.realpath(os.path.join(bids_root_path,
 #                                                 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
 #     sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))

hed/tools/analysis/hed_type_values.py

@@ -240,7 +240,7 @@ def _update_variables(self, tag_list, index):
 #     conditions2 = HedTypeValues(HedContextManager(test_strings2), hed_schema, defs)
 #     conditions3 = HedTypeValues(HedContextManager(test_strings3), hed_schema, defs)
 #     bids_root_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)),
-#                                                    '../../../tests/data/bids_tests/eeg_ds003654s_hed'))
+#                                                    '../../../tests/data/bids_tests/eeg_ds003645s_hed'))
 #     events_path = os.path.realpath(os.path.join(bids_root_path,
 #                                                 'sub-002/eeg/sub-002_task-FacePerception_run-1_events.tsv'))
 #     sidecar_path = os.path.realpath(os.path.join(bids_root_path, 'task-FacePerception_events.json'))

hed/tools/remodeling/operations/summarize_hed_tags_op.py

@@ -94,13 +94,12 @@ def __init__(self, sum_op):
         self.expand_context = sum_op.expand_context
 
     def update_context(self, new_context):
-        counts = HedTagCounts(new_context['name'])
+        counts = HedTagCounts(new_context['name'], total_events=len(new_context['df']))
         input_data = TabularInput(new_context['df'], hed_schema=new_context['schema'], sidecar=new_context['sidecar'])
         definitions = input_data.get_definitions().gathered_defs
         for objs in input_data.iter_dataframe(hed_ops=[new_context['schema']], return_string_only=False,
                                               expand_defs=False, remove_definitions=True):
-            counts.update_event_counts(objs['HED'])
-
+            counts.update_event_counts(objs['HED'], new_context['name'])
         self.summary_dict[new_context["name"]] = counts
 
     def _get_summary_details(self, merge_counts):
@@ -109,25 +108,64 @@ def _get_summary_details(self, merge_counts):
         for key, key_list in self.tags.items():
             details[key] = self._get_details(key_list, template, verbose=True)
         leftovers = [value.get_info(verbose=True) for value in unmatched]
-        return {"Main tags": details, "Other tags": leftovers}
+        return {"name": merge_counts.name, "total_events": merge_counts.total_events,
+                "files": [name for name in merge_counts.files.keys()],
+                "Main tags": details, "Other tags": leftovers}
 
     def _get_result_string(self, name, result, indent=BaseContext.DISPLAY_INDENT):
-        sum_list = [f"\n{name}\n{indent}Main tags[events,files]:"]
-        for category, tags in result['Main tags'].items():
-            sum_list.append(f"{indent}{indent}{category}:")
-            if tags:
-                sum_list.append(f"{indent}{indent}{indent}{' '.join(self._tag_details(tags))}")
-        if result['Other tags']:
-            sum_list.append(f"\tOther tags[events,files]:")
-            sum_list.append(f"{indent}{indent}{' '.join(self._tag_details(result['Other tags']))}")
-        return "\n".join(sum_list)
+        if name == 'Dataset':
+            return self._get_dataset_string(result, indent=indent)
+        return self._get_individual_string(name, result, indent=indent)
+        # sum_list = [f"\n{name}\n{indent}Main tags[events,files]:"]
+        # for category, tags in result['Main tags'].items():
+        #     sum_list.append(f"{indent}{indent}{category}:")
+        #     if tags:
+        #         sum_list.append(f"{indent}{indent}{indent}{' '.join(self._tag_details(tags))}")
+        # if result['Other tags']:
+        #     sum_list.append(f"\tOther tags[events,files]:")
+        #     sum_list.append(f"{indent}{indent}{' '.join(self._tag_details(result['Other tags']))}")
+        # return "\n".join(sum_list)
 
+    def _merge_all(self):
+        all_counts = HedTagCounts('Dataset')
+        for key, counts in self.summary_dict.items():
+            all_counts.merge_tag_dicts(counts.tag_dict)
+            for file_name in counts.files.keys():
+                all_counts.files[file_name] = ""
+            all_counts.total_events = all_counts.total_events + counts.total_events
+        return all_counts
+
+    @staticmethod
+    def _get_dataset_string(result, indent=BaseContext.DISPLAY_INDENT):
+        sum_list = [f"Dataset: {len(result.get('files', []))} files with a total of "
+                    f"{result.get('total_events', 0)} events"]
+        sum_list = sum_list + HedTagSummaryContext._get_tag_list(result, indent=indent)
+        return "\n".join(sum_list)
+
+    @staticmethod
+    def _get_individual_string(name, result, indent=BaseContext.DISPLAY_INDENT):
+        sum_list = [f"{name}: {result.get('total_events', 0)} events"]
+        sum_list = sum_list + HedTagSummaryContext._get_tag_list(result, indent=indent)
+        return "\n".join(sum_list)
+
     @staticmethod
     def _tag_details(tags):
         tag_list = []
         for tag in tags:
             tag_list.append(f"{tag['tag']}[{tag['events']},{len(tag['files'])}]")
         return tag_list
+
+    @staticmethod
+    def _get_tag_list(tag_info, indent=BaseContext.DISPLAY_INDENT):
+        sum_list = [f"\n{indent}Main tags[events,files]:"]
+        for category, tags in tag_info['Main tags'].items():
+            sum_list.append(f"{indent}{indent}{category}:")
+            if tags:
+                sum_list.append(f"{indent}{indent}{indent}{' '.join(HedTagSummaryContext._tag_details(tags))}")
+        if tag_info['Other tags']:
+            sum_list.append(f"{indent}Other tags[events,files]:")
+            sum_list.append(f"{indent}{indent}{' '.join(HedTagSummaryContext._tag_details(tag_info['Other tags']))}")
+        return sum_list
 
     @staticmethod
     def _get_details(key_list, template, verbose=False):
@@ -137,8 +175,4 @@ def _get_details(key_list, template, verbose=False):
                 key_details.append(tag_cnt.get_info(verbose=verbose))
         return key_details
 
-    def _merge_all(self):
-        all_counts = HedTagCounts()
-        for key, counts in self.summary_dict.items():
-            HedTagCounts.merge_tag_dicts(all_counts.tag_dict, counts.tag_dict)
-        return all_counts
+