Broken .get_performance_metrics() method in VisualBehaviorNeuropixelsProjectCache.from_s3_cache(cache_dir=neuropixel_dataset_behavior)

[RobertoDF]

Describe the bug

.get_performance_metrics() will lead to

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/pandas/core/indexes/base.py:3652, in Index.get_loc(self, key)
   3651 try:
-> 3652     return self._engine.get_loc(casted_key)
   3653 except KeyError as err:

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/pandas/_libs/index.pyx:147, in pandas._libs.index.IndexEngine.get_loc()

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/pandas/_libs/index.pyx:176, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:7080, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:7088, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'change_time'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In[7], line 1
----> 1 session.get_performance_metrics()

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/behavior_session.py:857, in BehaviorSession.get_performance_metrics(self, engaged_trial_reward_rate_threshold)
    838 performance_metrics = {
    839     "trial_count": self._trials.trial_count,
    840     "go_trial_count": self._trials.go_trial_count,
   (...)
    849     "total_reward_volume": self.rewards.volume.sum(),
    850 }
    851 # Although 'earned_reward_count' will currently have the same value as
    852 # 'hit_trial_count', in the future there may be variants of the
    853 # task where rewards are withheld. In that case the
    854 # 'earned_reward_count' will be smaller than (and different from)
    855 # the 'hit_trial_count'.
--> 857 rpdf = self.get_rolling_performance_df()
    858 engaged_trial_mask = (
    859     rpdf["reward_rate"] > engaged_trial_reward_rate_threshold
    860 )
    861 performance_metrics["maximum_reward_rate"] = np.nanmax(
    862     rpdf["reward_rate"].values
    863 )

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/behavior_session.py:754, in BehaviorSession.get_rolling_performance_df(self)
    718 def get_rolling_performance_df(self) -> pd.DataFrame:
    719     """Return a DataFrame containing trial by trial behavior response
    720     performance metrics.
    721 
   (...)
    752 
    753     """
--> 754     return self._trials.rolling_performance

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/data_objects/trials/trials.py:345, in Trials.rolling_performance(self)
    308 @property
    309 def rolling_performance(self) -> pd.DataFrame:
    310     """Return a DataFrame containing trial by trial behavior response
    311     performance metrics.
    312 
   (...)
    343 
    344     """
--> 345     reward_rate = self.calculate_reward_rate()
    347     # Indices to build trial metrics dataframe:
    348     trials_index = self.data.index

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/data_objects/trials/trials.py:455, in Trials.calculate_reward_rate(self, window, trial_window, initial_trials)
    452 def calculate_reward_rate(
    453     self, window=0.75, trial_window=25, initial_trials=10
    454 ):
--> 455     response_latency = self._calculate_response_latency_list()
    456     starttime = self.start_time.values
    457     assert len(response_latency) == len(starttime)

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/data_objects/trials/trials.py:433, in Trials._calculate_response_latency_list(self)
    411 def _calculate_response_latency_list(self) -> List:
    412     """per trial, determines a response latency
    413 
    414     Returns
   (...)
    430     difference).
    431     """
    432     df = pd.DataFrame(
--> 433         {"lick_times": self.lick_times, "change_time": self.change_time}
    434     )
    435     df["valid_response_licks"] = df.apply(
    436         lambda trial: [
    437             lt
   (...)
    441         axis=1,
    442     )
    443     response_latency = df.apply(
    444         lambda trial: trial["valid_response_licks"][0]
    445         - trial["change_time"]
   (...)
    448         axis=1,
    449     )

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/allensdk/brain_observatory/behavior/data_objects/trials/trials.py:278, in Trials.change_time(self)
    276 @property
    277 def change_time(self) -> pd.Series:
--> 278     return self.data["change_time"]

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/pandas/core/frame.py:3760, in DataFrame.__getitem__(self, key)
   3758 if self.columns.nlevels > 1:
   3759     return self._getitem_multilevel(key)
-> 3760 indexer = self.columns.get_loc(key)
   3761 if is_integer(indexer):
   3762     indexer = [indexer]

File /alzheimer/Roberto/Software/mambaforge/envs/De-Filippo-et-al-2023/lib/python3.9/site-packages/pandas/core/indexes/base.py:3654, in Index.get_loc(self, key)
   3652     return self._engine.get_loc(casted_key)
   3653 except KeyError as err:
-> 3654     raise KeyError(key) from err
   3655 except TypeError:
   3656     # If we have a listlike key, _check_indexing_error will raise
   3657     #  InvalidIndexError. Otherwise we fall through and re-raise
   3658     #  the TypeError.
   3659     self._check_indexing_error(key)

KeyError: 'change_time'

To Reproduce

cache = VisualBehaviorNeuropixelsProjectCache.from_s3_cache(cache_dir=neuropixel_dataset_behavior)
behavior_session_id = 1062781531#1064442478
session.get_performance_metrics()

Environment (please complete the following information):

• OS & version: Ubuntu
• Python version 3.9.16
• AllenSDK version 2.16

Additional context session.trials contains the column "change_time_no_display_delay" not "change_time"

[morriscb]

Hey Roberto, your PR should now be merged in the latest release version. Thanks for catching this.

[RobertoDF]

Great👍