Removes dependency on hardcoded acronym-id map and strips subregion from acronym

allensdk/brain_observatory/ecephys/_channel.py

@@ -15,14 +15,23 @@ def __init__(
             probe_vertical_position: int,
             probe_horizontal_position: int,
             manual_structure_acronym: str = '',
-            manual_structure_id: Optional[int] = None,
             anterior_posterior_ccf_coordinate: Optional[float] = None,
             dorsal_ventral_ccf_coordinate: Optional[float] = None,
             left_right_ccf_coordinate: Optional[float] = None,
             impedance: float = np.nan,
             filtering: str = 'AP band: 500 Hz high-pass; '
-                             'LFP band: 1000 Hz low-pass'
+                             'LFP band: 1000 Hz low-pass',
+            strip_structure_subregion: bool = True
     ):
+        """
+
+        Parameters
+        ----------
+        strip_structure_subregion: Whether to remove the subregion from the
+            structure acronym. I.e if the acronym is "LGd-sh" then it will get
+            parsed as "LGd". You might want to strip it if the subregion is
+            beyond annotation accuracy.
+        """
         super().__init__(name='channel', value=self)
         self._id = id
         self._probe_id = probe_id
@@ -31,13 +40,13 @@ def __init__(
         self._probe_vertical_position = probe_vertical_position
         self._probe_horizontal_position = probe_horizontal_position
         self._manual_structure_acronym = manual_structure_acronym
-        self._manual_structure_id = manual_structure_id
         self._anterior_posterior_ccf_coordinate = \
             anterior_posterior_ccf_coordinate
         self._dorsal_ventral_ccf_coordinate = dorsal_ventral_ccf_coordinate
         self._left_right_ccf_coordinate = left_right_ccf_coordinate
         self._impedance = impedance
         self._filtering = filtering
+        self._strip_structure_subregion = strip_structure_subregion
 
     @property
     def id(self) -> int:
@@ -65,11 +74,9 @@ def probe_horizontal_position(self) -> int:
 
     @property
     def manual_structure_acronym(self) -> str:
-        return self._manual_structure_acronym
-
-    @property
-    def manual_structure_id(self) -> Optional[int]:
-        return self._manual_structure_id
+        return self._manual_structure_acronym.split('-')[0] \
+            if self._strip_structure_subregion \
+            else self._manual_structure_acronym
 
     @property
     def anterior_posterior_ccf_coordinate(self) -> Optional[float]:

allensdk/brain_observatory/ecephys/_channels.py

@@ -17,9 +17,29 @@ def __init__(self, channels: List[Channel]):
         super().__init__(name='channels', value=channels)
 
     @classmethod
-    def from_json(cls, channels: dict) -> "Channels":
-        channels = [Channel(**{f'{"impedance" if k == "impedence" else k}': v
-                               for k, v in channel.items()})
+    def from_json(
+            cls,
+            channels: dict
+    ) -> "Channels":
+        for channel in channels:
+            if 'impedence' in channel:
+                # Correct misspelling
+                channel['impedance'] = channel.pop('impedence')
+
+        channels = [Channel(
+            id=channel['id'],
+            probe_id=channel['probe_id'],
+            valid_data=channel['valid_data'],
+            local_index=channel['local_index'],
+            probe_vertical_position=channel['probe_vertical_position'],
+            probe_horizontal_position=channel['probe_horizontal_position'],
+            manual_structure_acronym=channel['manual_structure_acronym'],
+            anterior_posterior_ccf_coordinate=(
+                channel['anterior_posterior_ccf_coordinate']),
+            dorsal_ventral_ccf_coordinate=(
+                channel['dorsal_ventral_ccf_coordinate']),
+            left_right_ccf_coordinate=channel['left_right_ccf_coordinate']
+        )
                     for channel in channels]
         return Channels(channels=channels)
 
@@ -81,27 +101,8 @@ def from_nwb(cls, nwbfile: NWBFile,
                 probe_id=row['probe_id'],
                 valid_data=row['valid_data'],
                 manual_structure_acronym=manual_structure_acronym,
-                manual_structure_id=STRUCTURE_ACRONYM_ID_MAP.get(
-                    manual_structure_acronym, np.nan),
                 anterior_posterior_ccf_coordinate=row['x'],
                 dorsal_ventral_ccf_coordinate=row['y'],
                 left_right_ccf_coordinate=row['z']
             ))
         return Channels(channels=channels)
-
-
-STRUCTURE_ACRONYM_ID_MAP = {
-    "grey": 8, "SCig": 10, "SCiw": 17, "IGL": 27, "LT": 66, "VL": 81,
-    "MRN": 128, "LD": 155, "LGd": 170, "LGv": 178, "APN": 215, "LP": 218,
-    "RT": 262, "MB": 313, "SGN": 325, "BMAa": 327, "CA": 375, "CA1": 382,
-    "VISp": 385, "VISam": 394, "VISal": 402, "VISl": 409, "VISrl": 417,
-    "CA2": 423, "CA3": 463, "SUB": 502, "VISpm": 533, "TH": 549,
-    "NOT": 628, "COAa": 639, "COApm": 663, "VIS": 669, "CP": 672,
-    "OLF": 698, "OP": 706, "VPL": 718, "DG": 726, "VPM": 733, "ZI": 797,
-    "SCzo": 834, "SCsg": 842, "SCop": 851, "PF": 930, "PO": 1020,
-    "POL": 1029, "POST": 1037, "PP": 1044, "PPT": 1061, "MGd": 1072,
-    "MGv": 1079, "PRE": 1084, "MGm": 1088, "HPF": 1089,
-    "VISli": 312782574, "VISmma": 480149258, "VISmmp": 480149286,
-    "ProS": 484682470, "RPF": 549009203, "Eth": 560581551,
-    "PIL": 560581563, "PoT": 563807435, "IntG": 563807439
-}

allensdk/brain_observatory/ecephys/probes.py

@@ -13,23 +13,6 @@
     NwbReadableInterface, NwbWritableInterface
 
 
-def _get_structure_key(channels: pd.DataFrame) -> str:
-    """
-    Scan a channels dataframe to determine if the structure
-    column is 'ecephys_structure_id' or 'manual_structure_id',
-
-    Return the appropriate key.
-    """
-    candidate_list = ('ecephys_structure_id',
-                      'manual_structure_id')
-    for candidate in candidate_list:
-        if candidate in channels.columns:
-            return candidate
-    msg = (f"Could not find {candidate_list} in channels data frame. "
-           f"Columns present are {channels.columns}")
-    raise RuntimeError(msg)
-
-
 class Probes(DataObject, JsonReadableInterface, NwbReadableInterface,
              NwbWritableInterface):
     """Probes"""
@@ -104,7 +87,7 @@ def get_units_table(
         filter_by_validity
             Whether to filter out units without quality == "good"
         filter_out_of_brain_units
-            Whether to filter out units with missing ecephys_structure_id
+            Whether to filter out units with missing ecephys_structure_acronym
         amplitude_cutoff_maximum
             Filter units by this upper bound
         presence_ratio_minimum
@@ -131,10 +114,9 @@ def get_units_table(
                 ) for p in self.probes
             ])
 
-            structure_key = _get_structure_key(channels)
-
             if filter_out_of_brain_units:
-                channels = channels[~(channels[structure_key].isna())]
+                channels = channels[
+                    ~(channels['manual_structure_acronym'].isna())]
 
             # noinspection PyTypeChecker
             channel_ids = set(channels.index.values.tolist())

allensdk/test/brain_observatory/ecephys/test_probes.py

@@ -4,6 +4,7 @@
 import pytest
 from pynwb import NWBFile
 
+from allensdk.brain_observatory.ecephys._channel import Channel
 from allensdk.brain_observatory.ecephys.probes import Probes
 from allensdk.brain_observatory.ecephys.write_nwb.schemas import Probe
 
@@ -57,3 +58,42 @@ def test_skip_probes(self):
             probes=self.input_data, skip_probes=skip_probes)
         assert sorted([p.name for p in probes]) == \
                sorted([p for p in names if p not in skip_probes])
+
+    @pytest.mark.requires_bamboo
+    def test_units_from_structure_with_acronym(self):
+        """Checks that if there are channels with subregion in manual
+        structure id, that units detected from this region are still included
+        in units table"""
+        expected_n_units = self._probes_from_json.get_units_table().shape[0]
+
+        # Set the _manual_structure_acronym to something with a hyphen
+        for probe in self._probes_from_json.probes:
+            for channel in probe.channels.value:
+                if channel._manual_structure_acronym == 'MGd':
+                    channel._manual_structure_acronym = 'MGd-foo'
+        obtained_n_units = self._probes_from_json.get_units_table().shape[0]
+
+        assert expected_n_units == obtained_n_units
+
+
+@pytest.mark.parametrize('manual_structure_acronym', ('LGd-sh', 'LGd'))
+@pytest.mark.parametrize('strip_structure_subregion', (True, False))
+def test_probe_channels_strip_subregion(
+        manual_structure_acronym, strip_structure_subregion):
+    """Tests that subregion is stripped from manual structure acronym"""
+    c = Channel(
+        id=1,
+        local_index=1,
+        probe_vertical_position=1,
+        probe_horizontal_position=1,
+        probe_id=1,
+        valid_data=True,
+        manual_structure_acronym=manual_structure_acronym,
+        strip_structure_subregion=strip_structure_subregion
+    )
+    if strip_structure_subregion:
+        expected = 'LGd'
+    else:
+        expected = 'LGd-sh' if manual_structure_acronym == 'LGd-sh' else 'LGd'
+
+    assert c.manual_structure_acronym == expected