Skip to content

Commit

Permalink
Merge branch 'vbn_2022_dev' into rc/2.13.5
Browse files Browse the repository at this point in the history
  • Loading branch information
danielsf committed Jun 13, 2022
2 parents 6f74fdc + 92cd3a0 commit 5362360
Show file tree
Hide file tree
Showing 204 changed files with 16,659 additions and 3,651 deletions.
2 changes: 1 addition & 1 deletion allensdk/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def one(x):
return x
if xlen != 1:
raise OneResultExpectedError("Expected length one result, received: "
f"{x} results from queryr")
f"{x} results from query")
if isinstance(x, set):
return list(x)[0]
else:
Expand Down
34 changes: 34 additions & 0 deletions allensdk/api/queries/donors_queries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
from typing import List

import pandas as pd

from allensdk.internal.api import PostgresQueryMixin
from allensdk.internal.api.queries.utils import build_in_list_selector_query


def get_death_date_for_mouse_ids(
lims_connections: PostgresQueryMixin,
mouse_ids_list: List[int]
) -> pd.DataFrame:
"""
Parameters
----------
lims_connections:
mouse_ids_list: list of mouse ids
Returns
-------
Dataframe with columns:
- mouse id: int
- death on: datetime
"""
query = f"""
SELECT external_donor_name as mouse_id, death_on
FROM donors
{build_in_list_selector_query(
col='external_donor_name',
valid_list=mouse_ids_list
)}
"""
return lims_connections.select(query=query)
151 changes: 17 additions & 134 deletions allensdk/brain_observatory/behavior/behavior_ophys_experiment.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,18 @@
from typing import Optional

import numpy as np
import pandas as pd
from pynwb import NWBFile

from allensdk.brain_observatory.behavior.behavior_session import (
BehaviorSession)
from allensdk.brain_observatory.behavior.data_files import SyncFile
from allensdk.brain_observatory.behavior.data_files.eye_tracking_file import \
EyeTrackingFile
from allensdk.brain_observatory.behavior.data_files\
.rigid_motion_transform_file import \
RigidMotionTransformFile
from allensdk.brain_observatory.behavior.data_objects import \
BehaviorSessionId, StimulusTimestamps
BehaviorSessionId
from allensdk.brain_observatory.behavior.data_objects.cell_specimens \
.cell_specimens import \
CellSpecimens, EventsParams
from allensdk.brain_observatory.behavior.data_objects.eye_tracking\
.eye_tracking_table import \
EyeTrackingTable
from allensdk.brain_observatory.behavior.data_objects.eye_tracking\
.rig_geometry import \
RigGeometry as EyeTrackingRigGeometry
from allensdk.brain_observatory.behavior.data_objects.metadata \
.behavior_metadata.date_of_acquisition import \
DateOfAcquisitionOphys, DateOfAcquisition
Expand Down Expand Up @@ -65,8 +55,6 @@ def __init__(self,
cell_specimens: CellSpecimens,
metadata: BehaviorOphysMetadata,
motion_correction: MotionCorrection,
eye_tracking_table: Optional[EyeTrackingTable],
eye_tracking_rig_geometry: Optional[EyeTrackingRigGeometry],
date_of_acquisition: DateOfAcquisition):
super().__init__(
behavior_session_id=behavior_session._behavior_session_id,
Expand All @@ -80,16 +68,18 @@ def __init__(self,
stimulus_timestamps=behavior_session._stimulus_timestamps,
task_parameters=behavior_session._task_parameters,
trials=behavior_session._trials,
date_of_acquisition=date_of_acquisition
date_of_acquisition=date_of_acquisition,
eye_tracking_rig_geometry=(
behavior_session._eye_tracking_rig_geometry),
eye_tracking_table=behavior_session._eye_tracking

)

self._metadata = metadata
self._projections = projections
self._ophys_timestamps = ophys_timestamps
self._cell_specimens = cell_specimens
self._motion_correction = motion_correction
self._eye_tracking = eye_tracking_table
self._eye_tracking_rig_geometry = eye_tracking_rig_geometry

def to_nwb(self) -> NWBFile:
nwbfile = super().to_nwb(add_metadata=False)
Expand All @@ -99,9 +89,6 @@ def to_nwb(self) -> NWBFile:
self._cell_specimens.to_nwb(nwbfile=nwbfile,
ophys_timestamps=self._ophys_timestamps)
self._motion_correction.to_nwb(nwbfile=nwbfile)
self._eye_tracking.to_nwb(nwbfile=nwbfile)
if self._eye_tracking_rig_geometry is not None:
self._eye_tracking_rig_geometry.to_nwb(nwbfile=nwbfile)

return nwbfile
# ==================== class and utility methods ======================
Expand Down Expand Up @@ -146,48 +133,38 @@ def _get_motion_correction():
return MotionCorrection.from_data_file(
rigid_motion_transform_file=rigid_motion_transform_file)

def _get_eye_tracking_table(sync_file: SyncFile):
eye_tracking_file = EyeTrackingFile.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
eye_tracking_table = EyeTrackingTable.from_data_file(
data_file=eye_tracking_file,
sync_file=sync_file,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames
)
return eye_tracking_table

lims_db = db_connection_creator(
fallback_credentials=LIMS_DB_CREDENTIAL_MAP
)

behavior_session_id = BehaviorSessionId.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)

is_multiplane_session = _is_multi_plane_session()

meta = BehaviorOphysMetadata.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db,
is_multiplane=is_multiplane_session
)

sync_file = SyncFile.from_lims(db=lims_db,
ophys_experiment_id=ophys_experiment_id)
sync_file = SyncFile.from_lims(
db=lims_db,
behavior_session_id=behavior_session_id.value)

monitor_delay = calculate_monitor_delay(
sync_file=sync_file, equipment=meta.behavior_metadata.equipment)

stimulus_timestamps = StimulusTimestamps.from_sync_file(
sync_file=sync_file,
monitor_delay=monitor_delay)

behavior_session_id = BehaviorSessionId.from_lims(
db=lims_db, ophys_experiment_id=ophys_experiment_id)
date_of_acquisition = DateOfAcquisitionOphys.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)
behavior_session = BehaviorSession.from_lims(
lims_db=lims_db,
behavior_session_id=behavior_session_id.value,
stimulus_timestamps=stimulus_timestamps,
sync_file=sync_file,
monitor_delay=monitor_delay,
date_of_acquisition=date_of_acquisition
date_of_acquisition=date_of_acquisition,
skip_eye_tracking=skip_eye_tracking,
eye_tracking_z_threshold=eye_tracking_z_threshold,
eye_tracking_dilation_frames=eye_tracking_dilation_frames
)
if is_multiplane_session:
ophys_timestamps = OphysTimestampsMultiplane.from_sync_file(
Expand All @@ -211,13 +188,6 @@ def _get_eye_tracking_table(sync_file: SyncFile):
exclude_invalid_rois=exclude_invalid_rois
)
motion_correction = _get_motion_correction()
if skip_eye_tracking:
eye_tracking_table = None
eye_tracking_rig_geometry = None
else:
eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_lims(
ophys_experiment_id=ophys_experiment_id, lims_db=lims_db)

return BehaviorOphysExperiment(
behavior_session=behavior_session,
Expand All @@ -226,8 +196,6 @@ def _get_eye_tracking_table(sync_file: SyncFile):
metadata=meta,
projections=projections,
motion_correction=motion_correction,
eye_tracking_table=eye_tracking_table,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
date_of_acquisition=date_of_acquisition
)

Expand Down Expand Up @@ -279,11 +247,6 @@ def _is_multi_plane_session():
),
exclude_invalid_rois=exclude_invalid_rois
)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_nwb(
nwbfile=nwbfile)
eye_tracking_table = EyeTrackingTable.from_nwb(
nwbfile=nwbfile, z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames)
motion_correction = MotionCorrection.from_nwb(nwbfile=nwbfile)
is_multiplane_session = _is_multi_plane_session()
metadata = BehaviorOphysMetadata.from_nwb(
Expand All @@ -298,8 +261,6 @@ def _is_multi_plane_session():
return BehaviorOphysExperiment(
behavior_session=behavior_session,
cell_specimens=cell_specimens,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
eye_tracking_table=eye_tracking_table,
motion_correction=motion_correction,
metadata=metadata,
ophys_timestamps=ophys_timestamps,
Expand Down Expand Up @@ -348,17 +309,6 @@ def _get_motion_correction():
return MotionCorrection.from_data_file(
rigid_motion_transform_file=rigid_motion_transform_file)

def _get_eye_tracking_table(sync_file: SyncFile):
eye_tracking_file = EyeTrackingFile.from_json(
dict_repr=session_data)
eye_tracking_table = EyeTrackingTable.from_data_file(
data_file=eye_tracking_file,
sync_file=sync_file,
z_threshold=eye_tracking_z_threshold,
dilation_frames=eye_tracking_dilation_frames
)
return eye_tracking_table

sync_file = SyncFile.from_json(dict_repr=session_data)
is_multiplane_session = _is_multi_plane_session()
meta = BehaviorOphysMetadata.from_json(
Expand Down Expand Up @@ -395,13 +345,6 @@ def _get_eye_tracking_table(sync_file: SyncFile):
exclude_invalid_rois=exclude_invalid_rois
)
motion_correction = _get_motion_correction()
if skip_eye_tracking:
eye_tracking_table = None
eye_tracking_rig_geometry = None
else:
eye_tracking_table = _get_eye_tracking_table(sync_file=sync_file)
eye_tracking_rig_geometry = EyeTrackingRigGeometry.from_json(
dict_repr=session_data)

return BehaviorOphysExperiment(
behavior_session=behavior_session,
Expand All @@ -410,8 +353,6 @@ def _get_eye_tracking_table(sync_file: SyncFile):
metadata=meta,
projections=projections,
motion_correction=motion_correction,
eye_tracking_table=eye_tracking_table,
eye_tracking_rig_geometry=eye_tracking_rig_geometry,
date_of_acquisition=behavior_session._date_of_acquisition
)

Expand Down Expand Up @@ -688,64 +629,6 @@ def segmentation_mask_image(self) -> Image:
"""
return self._cell_specimens.segmentation_mask_image

@property
def eye_tracking(self) -> pd.DataFrame:
"""A dataframe containing ellipse fit parameters for the eye, pupil
and corneal reflection (cr). Fits are derived from tracking points
from a DeepLabCut model applied to video frames of a subject's
right eye. Raw tracking points and raw video frames are not exposed
by the SDK.
Notes:
- All columns starting with 'pupil_' represent ellipse fit parameters
relating to the pupil.
- All columns starting with 'eye_' represent ellipse fit parameters
relating to the eyelid.
- All columns starting with 'cr_' represent ellipse fit parameters
relating to the corneal reflection, which is caused by an infrared
LED positioned near the eye tracking camera.
- All positions are in units of pixels.
- All areas are in units of pixels^2
- All values are in the coordinate space of the eye tracking camera,
NOT the coordinate space of the stimulus display (i.e. this is not
gaze location), with (0, 0) being the upper-left corner of the
eye-tracking image.
- The 'likely_blink' column is True for any row (frame) where the pupil
fit failed OR eye fit failed OR an outlier fit was identified on the
pupil or eye fit.
- The pupil_area, cr_area, eye_area columns are set to NaN wherever
'likely_blink' == True.
- The pupil_area_raw, cr_area_raw, eye_area_raw columns contains all
pupil fit values (including where 'likely_blink' == True).
- All ellipse fits are derived from tracking points that were output by
a DeepLabCut model that was trained on hand-annotated data from a
subset of imaging sessions on optical physiology rigs.
- Raw DeepLabCut tracking points are not publicly available.
:rtype: pandas.DataFrame
"""
return self._eye_tracking.value

@property
def eye_tracking_rig_geometry(self) -> dict:
"""the eye tracking equipment geometry associate with a
given ophys experiment session.
Returns
-------
dict
dictionary with the following keys:
camera_eye_position_mm (array of float)
camera_rotation_deg (array of float)
equipment (string)
led_position (array of float)
monitor_position_mm (array of float)
monitor_rotation_deg (array of float)
"""
if self._eye_tracking_rig_geometry is None:
return dict()
return self._eye_tracking_rig_geometry.to_dict()['rig_geometry']

@property
def roi_masks(self) -> pd.DataFrame:
return self.cell_specimen_table[['cell_roi_id', 'roi_mask']]
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
from allensdk.brain_observatory.behavior.behavior_project_cache.\
project_apis.data_io import VisualBehaviorNeuropixelsProjectCloudApi
from allensdk.brain_observatory.behavior.behavior_project_cache.\
project_cache_base import ProjectCacheBase


class VisualBehaviorNeuropixelsProjectCache(ProjectCacheBase):

PROJECT_NAME = "visual-behavior-neuropixels-2022"
BUCKET_NAME = "sfd-cloudcache-test-bucket"

def __init__(
self,
fetch_api: VisualBehaviorNeuropixelsProjectCloudApi,
fetch_tries: int = 2,
):
""" Entrypoint for accessing Visual Behavior Neuropixels data.
Supports access to metadata tables:
get_ecephys_session_table()
get_behavior_session_table()
get_probe_table()
get_channel_table()
get_unit_table
Provides methods for instantiating session objects
from the nwb files:
get_ecephys_session() to load BehaviorEcephysSession
get_behavior_sesion() to load BehaviorSession
Provides tools for downloading data:
Will download data from the s3 bucket if session nwb file is not
in the local cache, othwerwise will use file from the cache.
"""
super().__init__(fetch_api=fetch_api, fetch_tries=fetch_tries)

@classmethod
def cloud_api_class(cls):
return VisualBehaviorNeuropixelsProjectCloudApi

def get_ecephys_session_table(self):
return self.fetch_api.get_ecephys_session_table(),

def get_behavior_session_table(self):
return self.fetch_api.get_behavior_session_table(),

def get_probe_table(self):
self.fetch_api.get_probe_table(),

def get_channel_table(self):
return self.fetch_api.get_channel_table(),

def get_unit_table(self):
return self.fetch_api.get_unit_table(),

def get_ecephys_session(self, ecephys_session_id: int):
return self.fetch_api.get_ecephys_session(ecephys_session_id)

def get_behavior_session(self, behavior_session_id: int):
return self.fetch_api.get_behavior_session(behavior_session_id)
Loading

0 comments on commit 5362360

Please sign in to comment.