key improperly restricting during spike sorting recording populate

[rpswenson]

Running this snippet:

test_key = {'nwb_file_name': 'RS1220240809_.nwb', 'sort_group_id': 0, 'interval_list_name': '02_r1', 
            'preproc_param_name': 'franklab_tetrode_hippocampus', 'team_name': 'mcoulter section'}
#sgs.SpikeSortingRecordingSelection() & key
sgs.SpikeSortingRecording.populate(key)

gives an output saying that an nwb is being written for a completely different animal, though in sgs.SpikeSortingRecordingSelection() the key above restricts to the correct entry. Here is the output and error:

[11:00:15][WARNING] Spyglass: Similar row(s) already inserted.
[11:02:07][INFO] Spyglass: Writing new NWB file peanut20201204_ZIZ07TR7MY.nwb

---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
Cell In[8], line 12
      4 key = {
      5     "nwb_file_name": nwb_file_name2,
      6     "sort_group_id": tetrode,
   (...)
      9     "team_name": "mcoulter section",
     10 }
     11 sgs.SpikeSortingRecordingSelection.insert_selection(key)
---> 12 sgs.SpikeSortingRecording.populate(key)
     14 key = (
     15     sgs.SpikeSortingRecordingSelection & {"nwb_file_name": nwb_file_name2} 
     16     & {"interval_list_name": interval} & {"sort_group_id": tetrode}).fetch1()
     18 sgs.ArtifactDetectionSelection.insert_selection({"recording_id": key["recording_id"], "artifact_param_name": "100uV_frac_07"})

File ~/spyglass/src/spyglass/utils/dj_mixin.py:612, in SpyglassMixin.populate(self, *restrictions, **kwargs)
    610 if use_transact:  # Pass single-process populate to super
    611     kwargs["processes"] = processes
--> 612     return super().populate(*restrictions, **kwargs)
    613 else:  # No transaction protection, use bare make
    614     for key in keys:

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/datajoint/autopopulate.py:248, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions)
    242 if processes == 1:
    243     for key in (
    244         tqdm(keys, desc=self.__class__.__name__)
    245         if display_progress
    246         else keys
    247     ):
--> 248         status = self._populate1(key, jobs, **populate_kwargs)
    249         if status is True:
    250             success_list.append(1)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/datajoint/autopopulate.py:315, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs)
    313 self.__class__._allow_insert = True
    314 try:
--> 315     make(dict(key), **(make_kwargs or {}))
    316 except (KeyboardInterrupt, SystemExit, Exception) as error:
    317     try:

File ~/spyglass/src/spyglass/spikesorting/v1/recording.py:192, in SpikeSortingRecording.make(self, key)
    190 sort_interval_valid_times = self._get_sort_interval_valid_times(key)
    191 recording, timestamps = self._get_preprocessed_recording(key)
--> 192 recording_nwb_file_name, recording_object_id = _write_recording_to_nwb(
    193     recording,
    194     timestamps,
    195     (SpikeSortingRecordingSelection & key).fetch1("nwb_file_name"),
    196 )
    197 key["analysis_file_name"] = recording_nwb_file_name
    198 key["object_id"] = recording_object_id

File ~/spyglass/src/spyglass/spikesorting/v1/recording.py:570, in _write_recording_to_nwb(recording, timestamps, nwb_file_name)
    566     nwbfile.add_acquisition(processed_electrical_series)
    567     recording_object_id = nwbfile.acquisition[
    568         "ProcessedElectricalSeries"
    569     ].object_id
--> 570     io.write(nwbfile)
    571 return analysis_nwb_file, recording_object_id

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5tools.py:375, in HDF5IO.write(self, **kwargs)
    370     raise UnsupportedOperation(("Cannot write to file %s in mode '%s'. "
    371                                 "Please use mode 'r+', 'w', 'w-', 'x', or 'a'")
    372                                % (self.source, self.__mode))
    374 cache_spec = popargs('cache_spec', kwargs)
--> 375 super().write(**kwargs)
    376 if cache_spec:
    377     self.__cache_spec()

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/io.py:99, in HDMFIO.write(self, **kwargs)
     97 """Write a container to the IO source."""
     98 f_builder = self.__manager.build(container, source=self.__source, root=True)
---> 99 self.write_builder(f_builder, **kwargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5tools.py:812, in HDF5IO.write_builder(self, **kwargs)
    809 self.logger.debug("Writing GroupBuilder '%s' to path '%s' with kwargs=%s"
    810                   % (f_builder.name, self.source, kwargs))
    811 for name, gbldr in f_builder.groups.items():
--> 812     self.write_group(self.__file, gbldr, **kwargs)
    813 for name, dbldr in f_builder.datasets.items():
    814     self.write_dataset(self.__file, dbldr, **kwargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5tools.py:994, in HDF5IO.write_group(self, **kwargs)
    991 if subgroups:
    992     for subgroup_name, sub_builder in subgroups.items():
    993         # do not create an empty group without attributes or links
--> 994         self.write_group(group, sub_builder, **kwargs)
    995 # write all datasets
    996 datasets = builder.datasets

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5tools.py:999, in HDF5IO.write_group(self, **kwargs)
    997 if datasets:
    998     for dset_name, sub_builder in datasets.items():
--> 999         self.write_dataset(group, sub_builder, **kwargs)
   1000 # write all links
   1001 links = builder.links

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/utils.py:664, in docval.<locals>.dec.<locals>.func_call(*args, **kwargs)
    662 def func_call(*args, **kwargs):
    663     pargs = _check_args(args, kwargs)
--> 664     return func(args[0], **pargs)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5tools.py:1318, in HDF5IO.write_dataset(self, **kwargs)
   1316 self.__set_written(builder)
   1317 if exhaust_dci:
-> 1318     self.__dci_queue.exhaust_queue()

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5_utils.py:72, in HDF5IODataChunkIteratorQueue.exhaust_queue(self)
     70 self.logger.debug("Exhausting DataChunkIterator from queue (length %d)" % len(self))
     71 dset, data = self.popleft()
---> 72 if self._write_chunk(dset, data):
     73     self.append(dataset=dset, data=data)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/backends/hdf5/h5_utils.py:53, in HDF5IODataChunkIteratorQueue._write_chunk(cls, dset, data)
     51 # Read the next data block
     52 try:
---> 53     chunk_i = next(data)
     54 except StopIteration:
     55     return False

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/hdmf/data_utils.py:367, in GenericDataChunkIterator.__next__(self)
    365 try:
    366     buffer_selection = next(self.buffer_selection_generator)
--> 367     return DataChunk(data=self._get_data(selection=buffer_selection), selection=buffer_selection)
    368 except StopIteration:
    369     if self.display_progress:

File ~/spyglass/src/spyglass/spikesorting/v1/recording.py:645, in SpikeInterfaceRecordingDataChunkIterator._get_data(self, selection)
    644 def _get_data(self, selection: Tuple[slice]) -> Iterable:
--> 645     return self.recording.get_traces(
    646         segment_index=self.segment_index,
    647         channel_ids=self.channel_ids[selection[1]],
    648         start_frame=selection[0].start,
    649         end_frame=selection[0].stop,
    650         return_scaled=self.return_scaled,
    651     )

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/baserecording.py:282, in BaseRecording.get_traces(self, segment_index, start_frame, end_frame, channel_ids, order, return_scaled, cast_unsigned)
    280 channel_indices = self.ids_to_indices(channel_ids, prefer_slice=True)
    281 rs = self._recording_segments[segment_index]
--> 282 traces = rs.get_traces(start_frame=start_frame, end_frame=end_frame, channel_indices=channel_indices)
    283 if order is not None:
    284     assert order in ["C", "F"]

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/preprocessing/filter.py:130, in FilterRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
    129 def get_traces(self, start_frame, end_frame, channel_indices):
--> 130     traces_chunk, left_margin, right_margin = get_chunk_with_margin(
    131         self.parent_recording_segment,
    132         start_frame,
    133         end_frame,
    134         channel_indices,
    135         self.margin,
    136         add_reflect_padding=self.add_reflect_padding,
    137     )
    139     traces_dtype = traces_chunk.dtype
    140     # if uint --> force int

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/recording_tools.py:232, in get_chunk_with_margin(rec_segment, start_frame, end_frame, channel_indices, margin, add_zeros, add_reflect_padding, window_on_margin, dtype)
    229     else:
    230         right_margin = margin
--> 232     traces_chunk = rec_segment.get_traces(
    233         start_frame - left_margin,
    234         end_frame + right_margin,
    235         channel_indices,
    236     )
    238 else:
    239     # either add_zeros or reflect_padding
    240     if start_frame is None:

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/channelslice.py:94, in ChannelSliceRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
     87 def get_traces(
     88     self,
     89     start_frame: Union[int, None] = None,
     90     end_frame: Union[int, None] = None,
     91     channel_indices: Union[list, None] = None,
     92 ) -> np.ndarray:
     93     parent_indices = self._parent_channel_indices[channel_indices]
---> 94     traces = self._parent_recording_segment.get_traces(start_frame, end_frame, parent_indices)
     95     return traces

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/preprocessing/common_reference.py:146, in CommonReferenceRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
    144 def get_traces(self, start_frame, end_frame, channel_indices):
    145     # need input trace
--> 146     all_traces = self.parent_recording_segment.get_traces(start_frame, end_frame, slice(None))
    147     all_traces = all_traces.astype(self.dtype)
    148     self.temp = np.zeros((all_traces.shape[0],), dtype=all_traces.dtype)

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/channelslice.py:94, in ChannelSliceRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
     87 def get_traces(
     88     self,
     89     start_frame: Union[int, None] = None,
     90     end_frame: Union[int, None] = None,
     91     channel_indices: Union[list, None] = None,
     92 ) -> np.ndarray:
     93     parent_indices = self._parent_channel_indices[channel_indices]
---> 94     traces = self._parent_recording_segment.get_traces(start_frame, end_frame, parent_indices)
     95     return traces

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/segmentutils.py:187, in ProxyConcatenateRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
    185     rec_seg = self.parent_segments[i0]
    186     seg_start = self.cumsum_length[i0]
--> 187     traces = rec_seg.get_traces(start_frame - seg_start, end_frame - seg_start, channel_indices)
    188 else:
    189     #  several segments
    190     all_traces = []

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/core/frameslicerecording.py:91, in FrameSliceRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
     89 parent_start = self.start_frame + start_frame
     90 parent_end = self.start_frame + end_frame
---> 91 traces = self._parent_recording_segment.get_traces(
     92     start_frame=parent_start, end_frame=parent_end, channel_indices=channel_indices
     93 )
     94 return traces

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/spikeinterface/extractors/nwbextractors.py:415, in NwbRecordingSegment.get_traces(self, start_frame, end_frame, channel_indices)
    413         traces = recordings[:, resorted_indices]
    414     else:
--> 415         traces = electrical_series_data[start_frame:end_frame, channel_indices]
    417 return traces

File h5py/_objects.pyx:54, in h5py._objects.with_phil.wrapper()

File h5py/_objects.pyx:55, in h5py._objects.with_phil.wrapper()

File ~/miniforge3/envs/spyglass/lib/python3.9/site-packages/h5py/_hl/dataset.py:758, in Dataset.__getitem__(self, args, new_dtype)
    756 if self._fast_read_ok and (new_dtype is None):
    757     try:
--> 758         return self._fast_reader.read(args)
    759     except TypeError:
    760         pass  # Fall back to Python read pathway below

File h5py/_selector.pyx:376, in h5py._selector.Reader.read()

OSError: Can't synchronously read data (wrong B-tree signature)

[CBroz1]

It was an unintentional byproduct of a recent update that populate, for no-transaction tables such as this, requires restriction by the primary key of the parent, otherwise all upstream keys are called. To determine which keys to use, you can look at Table.key_source.

You can track progress on addressing this issue in #1102