Add trials

Makefile

@@ -46,9 +46,7 @@ checkpdb:
 	find {src,tests} -name "*.py" -exec grep -Hn pdb {} \;
 
 devtest:
-	$(PYTHON) -W ignore:::pynwb.form.build.map: test.py
-	$(MAKE) flake
-	$(MAKE) checkpdb
+	$(PYTHON) -W ignore:::pynwb.form.build.map: test.py -fpi
 
 apidoc:
 	pip install -r requirements-doc.txt

docs/gallery/domain/icephys.py

@@ -46,8 +46,9 @@
 # :py:meth:`~pynwb.file.NWBFile.create_ic_electrode`.
 
 elec = nwbfile.create_ic_electrode(
-    name="elec0", source='', slice='', resistance='', seal='', description='',
-    location='', filtering='', initial_access_resistance='', device=device.name)
+    name="elec0", source='PyNWB tutorial example',
+    description='a mock intracellular electrode',
+    device=device.name)
 
 #######################
 # Stimulus data

docs/gallery/general/file.py

@@ -140,6 +140,33 @@
 nwbfile.create_epoch('the first epoch', 2.0, 4.0, ['first', 'example'], [test_ts, mod_ts])
 nwbfile.create_epoch('the second epoch', 6.0, 8.0, ['second', 'example'], [test_ts, mod_ts])
 
+####################
+# .. _basic_trials:
+#
+# Trials
+# ------
+#
+# Trials can be added to an NWB file using the methods :py:func:`~pynwb.file.NWBFile.add_trial`
+# and :py:func:`~pynwb.file.NWBFile.add_trial_column`. Together, these methods maintains a
+# table-like structure that can define arbitrary columns without having to go through the
+# extension process.
+#
+# By default, NWBFile only requires trial start time and trial end time. Additional columns
+# can be added using :py:func:`~pynwb.file.NWBFile.add_trial_column`. This method takes a name
+# for the column and a description of what the column stores. You do not need to supply data
+# type, as this will inferred.
+
+# Once all columns have been added, trial data can be populated using :py:func:`~pynwb.file.NWBFile.add_trial`.
+# This method takes a dict with keys that correspond to column names.
+#
+# Lets add an additional column and some trial data.
+
+nwbfile.add_trial_column('stim', 'the visual stimuli during the trial')
+
+nwbfile.add_trial({'start': 0, 'end': 2, 'stim': 'person'})
+nwbfile.add_trial({'start': 3, 'end': 5, 'stim': 'ocean'})
+nwbfile.add_trial({'start': 6, 'end': 8, 'stim': 'desert'})
+
 ####################
 # .. _basic_writing:
 #

requirements.txt

@@ -8,3 +8,4 @@ requests==2.18.4
 ruamel.yaml==0.15.37
 six==1.11.0
 urllib3==1.22
+pandas==0.19.2

setup.py

@@ -26,6 +26,7 @@
     'install_requires':
     [
         'numpy',
+        'pandas',
         'h5py',
         'ruamel.yaml',
         'python-dateutil',

src/pynwb/core.py

@@ -249,6 +249,8 @@ def __len__(self):
         return len(self.__data)
 
     def __getitem__(self, args):
+        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list)):
+            return [self.data[i] for i in args]
         return self.data[args]
 
     def append(self, arg):
@@ -720,3 +722,171 @@ def __build_class(cls, name, bases, classdict):
 
         if len(clsconf) == 1:
             setattr(cls, '__getitem__', cls.__make_getitem(attr, container_type))
+
+
+@register_class('TableColumn', CORE_NAMESPACE)
+class TableColumn(NWBData):
+
+    __nwbfields__ = (
+        'description',
+    )
+
+    @docval({'name': 'name', 'type': str, 'doc': 'the name of this column'},
+            {'name': 'description', 'type': str, 'doc': 'a description for this column'},
+            {'name': 'data', 'type': 'array_data', 'doc': 'the data contained in this  column', 'default': list()})
+    def __init__(self, **kwargs):
+        desc = popargs('description', kwargs)
+        call_docval_func(super(TableColumn, self).__init__, kwargs)
+        self.description = desc
+
+    @docval({'name': 'val', 'type': None, 'doc': 'the value to add to this column'})
+    def add_row(self, **kwargs):
+        val = getargs('val', kwargs)
+        self.data.append(val)
+
+
+@register_class('DynamicTable', CORE_NAMESPACE)
+class DynamicTable(NWBContainer):
+    """
+    A column-based table. Columns are defined by the argument *columns*. This argument
+    must be a list/tuple of TableColumns or a list/tuple of dicts containing the keys
+    'name' and 'description' that provide the name and description of each column
+    in the table.
+    """
+
+    __nwbfields__ = (
+        {'name': 'id', 'child': True},
+        {'name': 'columns', 'child': True},
+        'colnames',
+        'description'
+    )
+
+    @docval({'name': 'name', 'type': str, 'doc': 'the name of this table'},
+            {'name': 'source', 'type': str, 'doc': 'a description of where this table came from'},
+            {'name': 'description', 'type': str, 'doc': 'a description of what is in this table'},
+            {'name': 'ids', 'type': ('array_data', ElementIdentifiers), 'doc': 'the identifiers for this table',
+             'default': list()},
+            {'name': 'columns', 'type': (tuple, list), 'doc': 'the columns in this table', 'default': list()})
+    def __init__(self, **kwargs):
+        ids, columns, desc = popargs('ids', 'columns', 'description', kwargs)
+        call_docval_func(super(DynamicTable, self).__init__, kwargs)
+        self.description = desc
+
+        if not isinstance(ids, ElementIdentifiers):
+            self.id = ElementIdentifiers('id', data=ids)
+        else:
+            self.id = ids
+
+        if len(columns) > 0:
+            if isinstance(columns[0], dict):
+                columns = tuple(TableColumn(**d) for d in columns)
+            elif not isinstance(columns[0], TableColumn):
+                raise ValueError("'columns' must be a list of TableColumns or dicts")
+            if not all(len(c) == len(columns[0]) for c in columns):
+                raise ValueError("columns must be the same length")
+            ni = len(self.id)
+            nc = len(columns[0])
+            if ni != nc:
+                if ni != 0 and nc != 0:
+                    raise ValueError("must provide same number of ids as length of columns if specifying ids")
+                elif nc != 0:
+                    for i in range(nc):
+                        self.id.data.append(i)
+                elif nc != 0:
+                    raise ValueError("cannot provide ids with no rows")
+
+        # column names for convenience
+
+        self.colnames = tuple(col.name for col in columns)
+        self.columns = columns
+
+        # to make generating DataFrames and Series easier
+        self.__df_cols = [self.id] + list(self.columns)
+        self.__df_colnames = [self.id.name] + [c.name for c in self.columns]
+
+        # for bookkeeping
+        self.__colids = {name: i for i, name in enumerate(self.colnames)}
+
+    def __len__(self):
+        return len(self.id)
+
+    @docval({'name': 'data', 'type': dict, 'help': 'the data to put in this row'},
+            {'name': 'id', 'type': int, 'help': 'the ID for the row', 'default': None})
+    def add_row(self, **kwargs):
+        '''
+        Add a row to the table. If *id* is not provided, it will auto-increment.
+        '''
+        data = getargs('data', kwargs)
+        for k, v in data.items():
+            colnum = self.__colids[k]
+            self.columns[colnum].add_row(v)
+        self.id.data.append(len(self.id))
+
+    # # keeping this around in case anyone wants to resurrect it
+    # # this was used to return a numpy structured array. this does not
+    # # work across platforms (it breaks on windows). instead, return
+    # # tuples and lists of tuples
+    # def get_dtype(self, col):
+    #     x = col.data[0]
+    #     shape = get_shape(x)
+    #     shape = None if shape is None else shape
+    #     while hasattr(x, '__len__') and not isinstance(x, (text_type, binary_type)):
+    #         x = x[0]
+    #     t = type(x)
+    #     if t in (text_type, binary_type):
+    #         t = np.string_
+    #     return (col.name, t, shape)
+
+    @docval(*get_docval(TableColumn.__init__))
+    def add_column(self, **kwargs):
+        """
+        Add a column to this table. If data is provided, it must
+        contain the same number of rows as the current state of the table.
+        """
+        col = TableColumn(**kwargs)
+        self.add_child(col)
+        name, data = col.name, col.data
+        if len(data) != len(self.id):
+            raise ValueError("column must have the same number of rows as 'id'")
+        self.__colids[name] = len(self.columns)
+        self.fields['colnames'] = tuple(list(self.colnames)+[name])
+        self.fields['columns'] = tuple(list(self.columns)+[col])
+        self.__df_colnames.append(name)
+        self.__df_cols.append(col)
+
+    def __getitem__(self, key):
+        ret = None
+        if isinstance(key, tuple):
+            # index by row and column, return specific cell
+            arg1 = key[0]
+            arg2 = key[1]
+            if isinstance(arg2, str):
+                arg2 = self.__colids[arg2] + 1
+            ret = self.__df_cols[arg2][arg1]
+        else:
+            arg = key
+            if isinstance(arg, str):
+                # index by one string, return column
+                ret = tuple(self.__df_cols[self.__colids[arg]+1].data)
+                # # keeping this around in case anyone wants to resurrect it
+                # dt = self.get_dtype(ret)[1]
+                # ret = np.array(ret.data, dtype=dt)
+            elif isinstance(arg, int):
+                # index by int, return row
+                ret = tuple(col[arg] for col in self.__df_cols)
+                # # keeping this around in case anyone wants to resurrect it
+                # dt = [self.get_dtype(col) for col in self.__df_cols]
+                # ret = np.array([ret], dtype=dt)
+
+            elif isinstance(arg, (tuple, list)):
+                # index by a list of ints, return multiple rows
+                # # keeping this around in case anyone wants to resurrect it
+                # dt = [self.get_dtype(col) for col in self.__df_cols]
+                # ret = np.zeros((len(arg),), dtype=dt)
+                # for name, col in zip(self.__df_colnames, self.__df_cols):
+                #     ret[name] = col[arg]
+                ret = list()
+                for i in arg:
+                    ret.append(tuple(col[i] for col in self.__df_cols))
+
+        return ret

src/pynwb/data/nwb.base.yaml

@@ -5,6 +5,17 @@ datasets:
     name: help
   doc: The attributes specified here are included in all interfaces.
   neurodata_type_def: NWBData
+- attributes:
+  - doc: Value is 'One of many columns that can be added to a DynamicTable'
+    dtype: text
+    value: One of many columns that can be added to a DynamicTable
+    name: help
+  - doc: A short description of what this column stores
+    dtype: text
+    name: description
+  doc: The attributes specified here are included in all interfaces.
+  neurodata_type_def: TableColumn
+  neurodata_type_inc: NWBData
 - default_name: vector_data
   attributes:
   - doc: a help string
@@ -202,3 +213,43 @@ groups:
   doc: A NWBDataInterface for storing images that have some relationship
   neurodata_type_def: Images
   neurodata_type_inc: NWBDataInterface
+- attributes:
+  - doc: Value is 'A column-centric table'
+    dtype: text
+    name: help
+    value: A column-centric table
+  - doc: The names of the columns in this table. This should be used to specifying an order to the columns
+    dtype: text
+    name: colnames
+    shape:
+      - null
+  - doc: Description of what is in this dynamic table
+    dtype: text
+    name: description
+  datasets:
+    - name: id
+      neurodata_type_inc: ElementIdentifiers
+      doc: The unique identifier for the rows in this dynamic table
+      dtype: int
+      shape:
+        - null
+    - doc: The columns in this dynamic table
+      neurodata_type_inc: TableColumn
+      quantity: '*'
+  doc: A group containing multiple datasets that are aligned on the first dimension (Currently, this requirement
+    if left up to APIs to check and enforce). Apart from a column that contains unique identifiers for each row
+    there are no other required datasets. Users are free to add any number of TableColumn objects here. Table
+    functionality is already supported through compound types, which is analogous to storing an array-of-structs.
+    DynamicTable can be thought of as a struct-of-arrays. This provides an alternative structure to choose from
+    when optimizing storage for anticipated access patterns. Additionally, this type provides a way of creating a
+    table without having to define a compound type up front. Although this convenience may be attractive, users
+    should think carefully about how data will be accessed. DynamicTable is more appropriate for column-centric
+    access, whereas a dataset with a compound type would be more appropriate for row-centric access. Finally,
+    data size should also be taken into account. For small tables, performance loss may be an acceptable trade-off
+    for the flexibility of a DynamicTable. For example, DynamicTable was originally developed for storing trial
+    data and spike unit metadata. Both of these use cases are expected to produce relatively small tables, so
+    the spatial locality of multiple datasets present in a DynamicTable is not expected to have a significant performance
+    impact. Additionally, requirements of trial and unit metadata tables are sufficiently diverse that performance
+    implications can be overlooked in favor of usability.
+  neurodata_type_def: DynamicTable
+  neurodata_type_inc: NWBDataInterface

src/pynwb/data/nwb.epoch.yaml

@@ -76,6 +76,10 @@ groups:
     dtype: text
     name: help
     value: A general epoch object
+  groups:
+    - name: metadata
+      doc: a DynamicTable for storing metadata about epochs
+      neurodata_type_inc: DynamicTable
   datasets:
     - name: epochs
       doc: the EpochTable holding information about each Epoch

src/pynwb/data/nwb.file.yaml

@@ -83,6 +83,7 @@ groups:
     \ different start/stop times."
     neurodata_type_inc: Epochs
     name: epochs
+    quantity: '?'
   - doc: "The home for processing Modules. These modules perform intermediate analysis\
       \ of data that is necessary to perform before scientific analysis. Examples\
       \ include spike clustering, extracting position from tracking data, stitching\
@@ -339,6 +340,29 @@ groups:
       name: optophysiology
       quantity: '?'
     name: general
+  - doc: 'Data about experimental trials'
+    name: trials
+    neurodata_type_inc: DynamicTable
+    quantity: '?'
+    datasets:
+    - doc: The start time of each trial
+      attributes:
+        - name: description
+          value: the start time of each trial
+          dtype: text
+          doc: Value is 'the start time of each trial'
+      name: start
+      neurodata_type_inc: TableColumn
+      dtype: float
+    - doc: The end time of each trial
+      attributes:
+        - name: description
+          value: the end time of each trial
+          dtype: text
+          doc: Value is 'the end time of each trial'
+      name: end
+      neurodata_type_inc: TableColumn
+      dtype: float
   name: root
   neurodata_type_def: NWBFile
   neurodata_type_inc: NWBContainer

src/pynwb/data/nwb.icephys.yaml

@@ -194,7 +194,6 @@ groups:
   - doc: Name(s) of devices in general/devices
     dtype: text
     name: device
-    quantity: '?'
   - doc: Electrode specific filtering.
     dtype: text
     name: filtering