Merge branch 'ticket/PSB-171/dev' into rc/2.16.0

allensdk/brain_observatory/behavior/behavior_ophys_experiment.py

@@ -611,13 +611,17 @@ def cell_specimen_table(self) -> pd.DataFrame:
                     which image plane an ROI resides on. Overlapping
                     ROIs are stored on different mask image planes
                 max_corretion_down: (float)
-                    max motion correction in down direction in pixels
+                   max motion correction in down direction in pixels. Defines
+                   the motion border at the top of the image.
                 max_correction_left: (float)
-                    max motion correction in left direction in pixels
+                    max motion correction in left direction in pixels. Defines
+                    the motion border at the right of the image.
                 max_correction_right: (float)
-                    max motion correction in right direction in pixels
+                    max motion correction in right direction in pixels. Defines
+                    the motion border at the left of the image.
                 max_correction_up: (float)
-                    max motion correction in up direction in pixels
+                    max motion correction in up direction in pixels. Defines
+                    the motion border at the bottom of the image.
                 roi_mask: (array of bool)
                     an image array that displays the location of the
                     roi mask in the field of view

allensdk/brain_observatory/behavior/behavior_project_cache/project_apis/data_io/behavior_project_cloud_api.py

@@ -163,7 +163,7 @@ def _get_ophys_session_table(self):
         df["date_of_acquisition"] = pd.to_datetime(
             df["date_of_acquisition"], utc="True"
         )
-        df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+        df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
         self._ophys_session_table = df.set_index("ophys_session_id")
 
     def get_ophys_session_table(self) -> pd.DataFrame:
@@ -188,7 +188,7 @@ def _get_behavior_session_table(self):
         df["date_of_acquisition"] = pd.to_datetime(
             df["date_of_acquisition"], utc="True"
         )
-        df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+        df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
 
         self._behavior_session_table = df.set_index("behavior_session_id")
 
@@ -218,7 +218,7 @@ def _get_ophys_experiment_table(self):
         df["date_of_acquisition"] = pd.to_datetime(
             df["date_of_acquisition"], utc="True"
         )
-        df = enforce_df_int_typing(df, INTEGER_COLUMNS)
+        df = enforce_df_int_typing(df, INTEGER_COLUMNS, True)
         self._ophys_experiment_table = df.set_index("ophys_experiment_id")
 
     def _get_ophys_cells_table(self):

allensdk/brain_observatory/behavior/behavior_session.py

@@ -1149,7 +1149,7 @@ def stimulus_templates(self) -> Optional[pd.DataFrame]:
             return None
 
     @property
-    def stimulus_fingerprint_movie_template(self) -> Optional[pd.DataFrame]:
+    def stimulus_natural_movie_template(self) -> Optional[pd.DataFrame]:
         """Get stimulus templates movie for the behavior session.
 
         Returns None if no stimulus movie is available.

allensdk/brain_observatory/behavior/data_objects/stimuli/fingerprint_stimulus.py

@@ -85,7 +85,7 @@ def from_stimulus_file(
                     'stop_time': stop_time,
                     'start_frame': start_frame,
                     'end_frame': end_frame,
-                    'repeat': repeat,
+                    'movie_repeat': repeat,
                     'duration': stop_time - start_time
                 })
         table = pd.DataFrame(res)

allensdk/brain_observatory/behavior/data_objects/stimuli/presentations.py

@@ -92,7 +92,7 @@ def __init__(
                     "is_change",
                     "is_image_novel",
                     "omitted",
-                    "repeat",
+                    "movie_repeat",
                     "flashes_since_change",
                     "trials_id",
                 ],
@@ -104,7 +104,7 @@ def __init__(
                 "flashes_since_change",
                 "image_index",
                 "movie_frame_index",
-                "repeat",
+                "movie_repeat",
                 "stimulus_index",
             ],
         )
@@ -290,6 +290,8 @@ def from_stimulus_file(
             data, stimulus_timestamps.value
         )
         raw_stim_pres_df = raw_stim_pres_df.drop(columns=["index"])
+        raw_stim_pres_df = cls._check_for_errant_omitted_stimulus(
+            input_df=raw_stim_pres_df)
 
         # Fill in nulls for image_name
         # This makes two assumptions:
@@ -540,17 +542,37 @@ def _postprocess(
                     )
                 }
             )
-        # Check if the first entry in the DataFrame is an omitted stimulus.
-        # This shouldn't happen and likely reflects some sort of camstim error
-        # with appending frames to the omitted flash frame log. See
-        # explanation here:
-        # https://github.com/AllenInstitute/AllenSDK/issues/2577
-        if "omitted" in df.columns and len(df) > 0:
-            first_row = df.iloc[0]
+        df = cls._check_for_errant_omitted_stimulus(input_df=df)
+        return df
+
+    @staticmethod
+    def _check_for_errant_omitted_stimulus(
+        input_df: pd.DataFrame
+    ) -> pd.DataFrame:
+        """Check if the first entry in the DataFrame is an omitted stimulus.
+
+        This shouldn't happen and likely reflects some sort of camstim error
+        with appending frames to the omitted flash frame log. See
+        explanation here:
+        https://github.com/AllenInstitute/AllenSDK/issues/2577
+
+        Parameters
+        ----------/
+        input_df : DataFrame
+            Input stimulus table to check for "omitted" stimulus.
+
+        Returns
+        -------
+        modified_df : DataFrame
+            Dataframe with omitted stimulus removed from first row or if not
+            found, return input_df unmodified.
+        """
+        if "omitted" in input_df.columns and len(input_df) > 0:
+            first_row = input_df.iloc[0]
             if not pd.isna(first_row["omitted"]):
                 if first_row["omitted"]:
-                    df = df.drop(first_row.name, axis=0)
-        return df
+                    input_df = input_df.drop(first_row.name, axis=0)
+        return input_df
 
     @staticmethod
     def _fill_missing_values_for_omitted_flashes(

allensdk/core/dataframe_utils.py

@@ -126,7 +126,8 @@ def enforce_df_column_order(
 
 
 def enforce_df_int_typing(input_df: pd.DataFrame,
-                          int_columns: List[str]) -> pd.DataFrame:
+                          int_columns: List[str],
+                          use_pandas_type=False) -> pd.DataFrame:
     """Enforce integer typing for columns that may have lost int typing when
     combined into the final DataFrame.
 
@@ -138,6 +139,10 @@ def enforce_df_int_typing(input_df: pd.DataFrame,
         Columns to enforce int typing and fill any NaN/None values with the
         value set in INT_NULL in this file. Requested columns not in the
         dataframe are ignored.
+    use_pandas_type : bool
+        Instead of filling with the value INT_NULL to enforce integer typing,
+        use the pandas type Int64. This type can have issues converting to
+        numpy/array type values.
 
     Returns
     -------
@@ -147,6 +152,9 @@ def enforce_df_int_typing(input_df: pd.DataFrame,
     """
     for col in int_columns:
         if col in input_df.columns:
-            input_df[col] = \
-                input_df[col].fillna(INT_NULL).astype(int)
+            if use_pandas_type:
+                input_df[col] = input_df[col].astype('Int64')
+            else:
+                input_df[col] = \
+                    input_df[col].fillna(INT_NULL).astype(int)
     return input_df

allensdk/test/brain_observatory/behavior/behavior_project_cache/test_behavior_project_cloud_api.py

@@ -108,7 +108,7 @@ def mock_cache(request, tmpdir):
             "ophys_session_table": pd.DataFrame(
                 {
                     "ophys_session_id": pd.Series([10, 11, 12, 13],
-                                                  dtype='int64'),
+                                                  dtype='Int64'),
                     "mouse_id": ["1"] * 4,
                     "date_of_acquisition": pd.to_datetime(["2021-01-01"] * 4),
                     "ophys_experiment_id": [4, 5, 6, [7, 8, 9]],

allensdk/test/brain_observatory/behavior/test_behavior_ophys_experiment.py

@@ -277,7 +277,7 @@ def test_behavior_ophys_experiment_list_data_attributes_and_methods(
         "segmentation_mask_image",
         "stimulus_presentations",
         "stimulus_templates",
-        'stimulus_fingerprint_movie_template',
+        'stimulus_natural_movie_template',
         "stimulus_timestamps",
         "task_parameters",
         "trials",

allensdk/test/brain_observatory/behavior/test_behavior_session.py

@@ -88,7 +88,7 @@ def dummy_init(self):
         'running_speed',
         'stimulus_presentations',
         'stimulus_templates',
-        'stimulus_fingerprint_movie_template',
+        'stimulus_natural_movie_template',
         'stimulus_timestamps',
         'task_parameters',
         'trials',