Merge pull request #1378 from AllenInstitute/rc/1.6.0

rc/1.6.0

CHANGELOG.md

@@ -1,6 +1,24 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [1.6.0] = 2020-03-23
+
+### Added
+- tutorial for optotagging for ecephys notebook
+- get\_receptive\_field() method in ecephys receptive field mapping
+
+### Changed
+- remove redundant sham\_change column in behavior sessions.trials table
+- versions for NWB output for ecephys and ophys behavior.
+- monitor delay is now calculated for BehaviorOphysLimsApi rather than defaulting to 0.0351
+
+### Bug Fixes
+- Fixed a bug where auto-rewarded trials were not properly attributed in the rewards property of a visual behavior
+- return None rather than raise exception if no container id was returned from lims id for given ophys id
+- Project caches no longer accept arbitrary keywords
+- matplotloib.pyplot.hist parameter normed no longer supported
+
+
 ## [1.5.0] = 2020-02-10
 
 ### Added

allensdk/__init__.py

@@ -36,7 +36,7 @@
 import logging
 
 
-__version__ = '1.5.1'
+__version__ = '1.6.0'
 
 
 try:

allensdk/brain_observatory/behavior/behavior_project_cache.py

@@ -2,7 +2,8 @@
 import os.path
 import csv
 from functools import partial
-from typing import Type, Callable, Optional, List, Any, Dict
+from typing import Type, Optional, List, Any, Dict, Union
+from pathlib import Path
 import pandas as pd
 import time
 import logging
@@ -15,8 +16,7 @@
     import BehaviorProjectBase
 from allensdk.api.caching_utilities import one_file_call_caching, call_caching
 from allensdk.core.exceptions import MissingDataError
-from allensdk.core.auth_config import LIMS_DB_CREDENTIAL_MAP
-from allensdk.core.authentication import credential_injector, DbCredentials
+from allensdk.core.authentication import DbCredentials
 
 BehaviorProjectApi = Type[BehaviorProjectBase]
 
@@ -64,11 +64,17 @@ def __init__(
             self,
             fetch_api: Optional[BehaviorProjectApi] = None,
             fetch_tries: int = 2,
-            **kwargs):
+            manifest: Optional[Union[str, Path]] = None,
+            version: Optional[str] = None,
+            cache: bool = True):
         """ Entrypoint for accessing visual behavior data. Supports
         access to summaries of session data and provides tools for
         downloading detailed session data (such as dff traces).
 
+        Likely you will want to use a class constructor, such as `from_lims`,
+        to initialize a BehaviorProjectCache, rather than calling this
+        directly.
+
         --- NOTE ---
         Because NWB files are not currently supported for this project (as of
         11/2019), this cache will not actually save any files of session data
@@ -87,38 +93,88 @@ def __init__(
             Used to pull data from remote sources, after which it is locally
             cached. Any object inheriting from BehaviorProjectBase is
             suitable. Current options are:
-                EcephysProjectLimsApi :: Fetches bleeding-edge data from the
+                BehaviorProjectLimsApi :: Fetches bleeding-edge data from the
                     Allen Institute"s internal database. Only works if you are
                     on our internal network.
         fetch_tries :
             Maximum number of times to attempt a download before giving up and
-            raising an exception. Note that this is total tries, not retries
-        **kwargs :
-            manifest : str or Path
-                full path at which manifest json will be stored
-            version : str
-                version of manifest file. If this mismatches the version
-                recorded in the file at manifest, an error will be raised.
-            other kwargs are passed to allensdk.api.cache.Cache
+            raising an exception. Note that this is total tries, not retries.
+            Default=2.
+        manifest : str or Path
+            full path at which manifest json will be stored. Defaults
+            to "behavior_project_manifest.json" in the local directory.
+        version : str
+            version of manifest file. If this mismatches the version
+            recorded in the file at manifest, an error will be raised.
+            Defaults to the manifest version in the class.
+        cache : bool
+            Whether to write to the cache. Default=True.
         """
-        kwargs["manifest"] = kwargs.get("manifest",
-                                        "behavior_project_manifest.json")
-        kwargs["version"] = kwargs.get("version", self.MANIFEST_VERSION)
+        manifest_ = manifest or "behavior_project_manifest.json"
+        version_ = version or self.MANIFEST_VERSION
 
-        super().__init__(**kwargs)
-        self.fetch_api = fetch_api or BehaviorProjectLimsApi.default()
+        super().__init__(manifest=manifest_, version=version_, cache=cache)
+        self.fetch_api = fetch_api
         self.fetch_tries = fetch_tries
         self.logger = logging.getLogger(self.__class__.__name__)
 
     @classmethod
-    def from_lims(cls, lims_credentials: Optional[DbCredentials] = None,
+    def from_lims(cls, manifest: Optional[Union[str, Path]] = None,
+                  version: Optional[str] = None,
+                  cache: bool = True,
+                  fetch_tries: int = 2,
+                  lims_credentials: Optional[DbCredentials] = None,
                   mtrain_credentials: Optional[DbCredentials] = None,
-                  app_kwargs: Dict[str, Any] = None, **kwargs):
-        return cls(fetch_api=BehaviorProjectLimsApi.default(
+                  host: Optional[str] = None,
+                  scheme: Optional[str] = None,
+                  asynchronous: bool = True) -> "BehaviorProjectCache":
+        """
+        Construct a BehaviorProjectCache with a lims api. Use this method
+        to create a  BehaviorProjectCache instance rather than calling
+        BehaviorProjectCache directly.
+
+        Parameters
+        ==========
+        manifest : str or Path
+            full path at which manifest json will be stored
+        version : str
+            version of manifest file. If this mismatches the version
+            recorded in the file at manifest, an error will be raised.
+        cache : bool
+            Whether to write to the cache
+        fetch_tries : int
+            Maximum number of times to attempt a download before giving up and
+            raising an exception. Note that this is total tries, not retries
+        lims_credentials : DbCredentials
+            Optional credentials to access LIMS database.
+            If not set, will look for credentials in environment variables.
+        mtrain_credentials: DbCredentials
+            Optional credentials to access mtrain database.
+            If not set, will look for credentials in environment variables.
+        host : str
+            Web host for the app_engine. Currently unused. This argument is
+            included for consistency with EcephysProjectCache.from_lims.
+        scheme : str
+            URI scheme, such as "http". Currently unused. This argument is
+            included for consistency with EcephysProjectCache.from_lims.
+        asynchronous : bool
+            Whether to fetch from web asynchronously. Currently unused.
+        Returns
+        =======
+        BehaviorProjectCache
+            BehaviorProjectCache instance with a LIMS fetch API
+        """
+        if host and scheme:
+            app_kwargs = {"host": host, "scheme": scheme,
+                          "asynchronous": asynchronous}
+        else:
+            app_kwargs = None
+        fetch_api = BehaviorProjectLimsApi.default(
                         lims_credentials=lims_credentials,
                         mtrain_credentials=mtrain_credentials,
-                        app_kwargs=app_kwargs),
-                   **kwargs)
+                        app_kwargs=app_kwargs)
+        return cls(fetch_api=fetch_api, manifest=manifest, version=version,
+                   cache=cache, fetch_tries=fetch_tries)
 
     def get_session_table(
             self,

allensdk/brain_observatory/behavior/rewards_processing.py

@@ -3,15 +3,15 @@
 
 
 def get_rewards(data, stimulus_rebase_function):
-    trial_df = pd.DataFrame(data["items"]["behavior"]['trial_log'])
-    rewards_dict = {'volume': [], 'timestamps': [], 'autorewarded': []}
+    trial_df = pd.DataFrame(data["items"]["behavior"]["trial_log"])
+    rewards_dict = {"volume": [], "timestamps": [], "autorewarded": []}
     for idx, trial in trial_df.iterrows():
         rewards = trial["rewards"]  # as i write this there can only ever be one reward per trial
         if rewards:
             rewards_dict["volume"].append(rewards[0][0])
             rewards_dict["timestamps"].append(stimulus_rebase_function(rewards[0][1]))
-            rewards_dict["autorewarded"].append('auto_rewarded' in trial['trial_params'])
+            rewards_dict["autorewarded"].append(trial["trial_params"]["auto_reward"])
 
-    df = pd.DataFrame(rewards_dict).set_index('timestamps', drop=True)
+    df = pd.DataFrame(rewards_dict).set_index("timestamps", drop=True)
 
     return df

allensdk/brain_observatory/behavior/trials_processing.py

@@ -345,6 +345,7 @@ def get_trials(data, licks_df, rewards_df, stimulus_presentations_df, rebase):
 
     trials = pd.DataFrame(all_trial_data).set_index('trial')
     trials.index = trials.index.rename('trials_id')
+    del trials["sham_change"]
 
     return trials
 

allensdk/brain_observatory/ecephys/copy_utility/__main__.py

@@ -12,11 +12,17 @@
 from allensdk.brain_observatory.argschema_utilities import write_or_print_outputs
 
 
-def hash_file(path, hasher_cls):
-    with open(path, 'rb') as file_obj:
-        hasher = hasher_cls()
-        hasher.update(file_obj.read())
-        return hasher.digest()
+def hash_file(path, hasher_cls, blocks_per_chunk=128):
+    """
+
+    """
+    hasher = hasher_cls()
+    with open(path, 'rb') as f:
+        # TODO: Update to new assignment syntax if drop < python 3.8 support
+        for chunk in iter(
+                lambda: f.read(hasher.block_size*blocks_per_chunk), b""):
+            hasher.update(chunk)
+    return hasher.digest()
 
 
 def walk_fs_tree(root, fn):