Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ticket/PSB-249: Address comments on final release candidate. #2728

Merged
merged 2 commits into from
Nov 1, 2023
Merged

Ticket/PSB-249: Address comments on final release candidate. #2728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f1ae5ab
Address comments on release from Marina
morriscb Oct 17, 2023
0620446
isort+black
morriscb Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 131 additions & 90 deletions allensdk/brain_observatory/behavior/behavior_project_cache/behavior_project_cache.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,39 @@
from typing import Optional, List, Union
import warnings
from pathlib import Path
import pandas as pd
import numpy as np
from typing import List, Optional, Union

import numpy as np
import pandas as pd
from allensdk.api.warehouse_cache.cache import Cache
from allensdk.brain_observatory.behavior.behavior_ophys_experiment import \
BehaviorOphysExperiment
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.experiments_table import \
ExperimentsTable
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.sessions_table import \
SessionsTable
from allensdk.api.warehouse_cache.caching_utilities import (
one_file_call_caching,
)
from allensdk.brain_observatory.behavior.behavior_ophys_experiment import (
BehaviorOphysExperiment,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.project_apis.data_io import ( # noqa: E501
BehaviorProjectLimsApi, BehaviorProjectCloudApi)
from allensdk.api.warehouse_cache.caching_utilities import \
one_file_call_caching
from allensdk.brain_observatory.behavior.behavior_project_cache.tables \
.ophys_sessions_table import \
BehaviorOphysSessionsTable
from allensdk.brain_observatory.behavior.behavior_session import \
BehaviorSession
from allensdk.brain_observatory.behavior.behavior_project_cache \
.project_cache_base import ProjectCacheBase
BehaviorProjectCloudApi,
BehaviorProjectLimsApi,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.project_cache_base import ( # noqa: E501
ProjectCacheBase,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.experiments_table import ( # noqa: E501
ExperimentsTable,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.ophys_sessions_table import ( # noqa: E501
BehaviorOphysSessionsTable,
)
from allensdk.brain_observatory.behavior.behavior_project_cache.tables.sessions_table import ( # noqa: E501
SessionsTable,
)
from allensdk.brain_observatory.behavior.behavior_session import (
BehaviorSession,
)


class UpdatedStimulusPresentationTableWarning(UserWarning):
pass


class VBOLimsCache(Cache):
Expand All @@ -41,41 +52,41 @@ class VBOLimsCache(Cache):
OPHYS_SESSIONS_KEY: {
"spec": f"{OPHYS_SESSIONS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
BEHAVIOR_SESSIONS_KEY: {
"spec": f"{BEHAVIOR_SESSIONS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
OPHYS_EXPERIMENTS_KEY: {
"spec": f"{OPHYS_EXPERIMENTS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
"typename": "file",
},
OPHYS_CELLS_KEY: {
"spec": f"{OPHYS_CELLS_KEY}.csv",
"parent_key": "BASEDIR",
"typename": "file"
}
"typename": "file",
},
}


class VisualBehaviorOphysProjectCache(ProjectCacheBase):

PROJECT_NAME = "visual-behavior-ophys"
BUCKET_NAME = "visual-behavior-ophys-data"
BUCKET_NAME = "staging.visual-behavior-ophys-data"

def __init__(
self,
fetch_api: Optional[Union[BehaviorProjectLimsApi,
BehaviorProjectCloudApi]] = None,
fetch_tries: int = 2,
manifest: Optional[Union[str, Path]] = None,
version: Optional[str] = None,
cache: bool = True
self,
fetch_api: Optional[
Union[BehaviorProjectLimsApi, BehaviorProjectCloudApi]
] = None,
fetch_tries: int = 2,
manifest: Optional[Union[str, Path]] = None,
version: Optional[str] = None,
cache: bool = True,
):
""" Entrypoint for accessing visual behavior data. Supports
"""Entrypoint for accessing visual behavior data. Supports
access to summaries of session data and provides tools for
downloading detailed session data (such as dff traces).

Expand Down Expand Up @@ -117,9 +128,26 @@ def __init__(

if not isinstance(self.fetch_api, BehaviorProjectCloudApi):
if cache:
self.cache = VBOLimsCache(manifest=manifest_,
version=version,
cache=cache)
self.cache = VBOLimsCache(
manifest=manifest_, version=version, cache=cache
)

warnings.warn(
message="\n\tAs of AllenSDK version 2.16.0, the latest Visual "
"Behavior Ophys data has been significantly updated from "
"previous releases. Specifically the user will need to "
"update all processing of the stimulus_presentations "
"tables. These tables now include multiple stimulus types "
"delineated by the columns `stimulus_block` and "
"`stimulus_block_name`.\n\nThe data that was available in "
"previous releases are stored in the "
"block name containing 'change_detection' and can be "
"accessed in the pandas table by using: \n\t"
"`stimulus_presentations["
"stimulus_presentations.stimulus_block_name.str.contains"
"('change_detection')]`",
category=UpdatedStimulusPresentationTableWarning,
)

@classmethod
def cloud_api_class(cls):
Expand All @@ -130,11 +158,11 @@ def lims_api_class(cls):
return BehaviorProjectLimsApi

def get_ophys_session_table(
self,
suppress: Optional[List[str]] = None,
index_column: str = "ophys_session_id",
as_df=True,
include_behavior_data=True
self,
suppress: Optional[List[str]] = None,
index_column: str = "ophys_session_id",
as_df=True,
include_behavior_data=True,
) -> Union[pd.DataFrame, BehaviorOphysSessionsTable]:
"""
Return summary table of all ophys_session_ids in the database.
Expand All @@ -155,13 +183,15 @@ def get_ophys_session_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_SESSIONS_KEY)
path = self.cache.get_cache_path(
None, self.cache.OPHYS_SESSIONS_KEY
)
ophys_sessions = one_file_call_caching(
path,
self.fetch_api.get_ophys_session_table,
_write_json,
lambda path: _read_json(path, index_name='ophys_session_id'))
lambda path: _read_json(path, index_name="ophys_session_id"),
)
else:
ophys_sessions = self.fetch_api.get_ophys_session_table()

Expand All @@ -171,24 +201,23 @@ def get_ophys_session_table(
suppress=suppress,
as_df=True,
include_ophys_data=False,
include_trial_metrics=False
include_trial_metrics=False,
)
ophys_sessions = behavior_sessions_table.merge(
ophys_sessions,
left_index=True,
right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
right_on="behavior_session_id",
suffixes=("_behavior", "_ophys"),
)

sessions = BehaviorOphysSessionsTable(df=ophys_sessions,
suppress=suppress,
index_column=index_column)
sessions = BehaviorOphysSessionsTable(
df=ophys_sessions, suppress=suppress, index_column=index_column
)

return sessions.table if as_df else sessions

def get_ophys_experiment_table(
self,
suppress: Optional[List[str]] = None,
as_df=True
self, suppress: Optional[List[str]] = None, as_df=True
) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all ophys_experiment_ids in the database.
Expand All @@ -201,27 +230,34 @@ def get_ophys_experiment_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_experiment_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHYS_EXPERIMENTS_KEY)
path = self.cache.get_cache_path(
None, self.cache.OPHYS_EXPERIMENTS_KEY
)
experiments = one_file_call_caching(
path,
self.fetch_api.get_ophys_experiment_table,
_write_json,
lambda path: _read_json(path,
index_name='ophys_experiment_id'))
lambda path: _read_json(
path, index_name="ophys_experiment_id"
),
)
else:
experiments = self.fetch_api.get_ophys_experiment_table()

# Merge behavior data in
behavior_sessions_table = self.get_behavior_session_table(
suppress=suppress, as_df=True, include_ophys_data=False,
include_trial_metrics=False
suppress=suppress,
as_df=True,
include_ophys_data=False,
include_trial_metrics=False,
)
experiments = behavior_sessions_table.merge(
experiments, left_index=True, right_on='behavior_session_id',
suffixes=('_behavior', '_ophys'))
experiments = ExperimentsTable(df=experiments,
suppress=suppress)
experiments,
left_index=True,
right_on="behavior_session_id",
suffixes=("_behavior", "_ophys"),
)
experiments = ExperimentsTable(df=experiments, suppress=suppress)
return experiments.table if as_df else experiments

def get_ophys_cells_table(self) -> pd.DataFrame:
Expand All @@ -232,25 +268,24 @@ def get_ophys_cells_table(self) -> pd.DataFrame:
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_ophys_cells_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.OPHyS_CELLS_KEY)
path = self.cache.get_cache_path(None, self.cache.OPHyS_CELLS_KEY)
ophys_cells_table = one_file_call_caching(
path,
self.fetch_api.get_ophys_cells_table,
_write_json,
lambda path: _read_json(path,
index_name='cell_roi_id'))
lambda path: _read_json(path, index_name="cell_roi_id"),
)
else:
ophys_cells_table = self.fetch_api.get_ophys_cells_table()

return ophys_cells_table

def get_behavior_session_table(
self,
suppress: Optional[List[str]] = None,
as_df=True,
include_ophys_data=True,
include_trial_metrics: bool = False
self,
suppress: Optional[List[str]] = None,
as_df=True,
include_ophys_data=True,
include_trial_metrics: bool = False,
) -> Union[pd.DataFrame, SessionsTable]:
"""
Return summary table of all behavior_session_ids in the database.
Expand All @@ -269,22 +304,23 @@ def get_behavior_session_table(
if isinstance(self.fetch_api, BehaviorProjectCloudApi):
return self.fetch_api.get_behavior_session_table()
if self.cache is not None:
path = self.cache.get_cache_path(None,
self.cache.BEHAVIOR_SESSIONS_KEY)
path = self.cache.get_cache_path(
None, self.cache.BEHAVIOR_SESSIONS_KEY
)
sessions = one_file_call_caching(
path,
self.fetch_api.get_behavior_session_table,
_write_json,
lambda path: _read_json(path,
index_name='behavior_session_id'))
lambda path: _read_json(
path, index_name="behavior_session_id"
),
)
else:
sessions = self.fetch_api.get_behavior_session_table()

if include_ophys_data:
ophys_session_table = self.get_ophys_session_table(
suppress=suppress,
as_df=False,
include_behavior_data=False
suppress=suppress, as_df=False, include_behavior_data=False
)
else:
ophys_session_table = None
Expand All @@ -293,12 +329,13 @@ def get_behavior_session_table(
suppress=suppress,
fetch_api=self.fetch_api,
ophys_session_table=ophys_session_table,
include_trial_metrics=include_trial_metrics)
include_trial_metrics=include_trial_metrics,
)

return sessions.table if as_df else sessions

def get_behavior_ophys_experiment(
self, ophys_experiment_id: int
self, ophys_experiment_id: int
) -> BehaviorOphysExperiment:
"""
Gets `BehaviorOphysExperiment` for `ophys_experiment_id`
Expand All @@ -311,11 +348,11 @@ def get_behavior_ophys_experiment(
BehaviorOphysExperiment
"""
return self.fetch_api.get_behavior_ophys_experiment(
ophys_experiment_id=ophys_experiment_id)
ophys_experiment_id=ophys_experiment_id
)

def get_behavior_session(
self,
behavior_session_id: int
self, behavior_session_id: int
) -> BehaviorSession:
"""
Gets `BehaviorSession` for `behavior_session_id`
Expand Down Expand Up @@ -346,7 +383,7 @@ def get_natural_movie_template(self, n_workers=None) -> pd.DataFrame:
frames as presented to the mouse. The DataFrame is indexed with the
same frame index as shown in the stimulus presentation table.

The processing of the movie requires signicant processing and its
The processing of the movie requires significant processing and its
return size is very large so take care in requesting this data.

Parameters
Expand Down Expand Up @@ -381,8 +418,12 @@ def _write_json(path, df):

def _read_json(path, index_name: Optional[str] = None):
"""Reads a dataframe file written to the cache by _write_json."""
df = pd.read_json(path, date_unit="s", orient="split",
convert_dates=["date_of_acquisition"])
df = pd.read_json(
path,
date_unit="s",
orient="split",
convert_dates=["date_of_acquisition"],
)
if index_name:
df = df.rename_axis(index=index_name)
return df
Loading
Loading