Broken process pool with reasonably sized sort interval

I get the following error when I try to populate SpikeSortingRecording for a large sort interval (about 10 hours long) as well as a more reasonably sized one (20 minues). Thanks in advance for any help with this.

write_binary_recording with n_jobs 4 chunk_size 156250000

/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/spec/namespace.py:532: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.5.0 because version 1.5.1 is already loaded. warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." /home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/spec/namespace.py:532: UserWarning: Ignoring cached namespace 'core' version 2.3.0 because version 2.4.0 is already loaded. warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." /home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/spec/namespace.py:532: UserWarning: Ignoring cached namespace 'hdmf-experimental' version 0.1.0 because version 0.2.0 is already loaded. warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." /home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/spec/namespace.py:532: UserWarning: Ignoring cached namespace 'ndx-franklab-novela' version 0.0.011.36 because version 0.1.0 is already loaded. warn("Ignoring cached namespace '%s' version %s because version %s is already loaded." Exception in initializer: Traceback (most recent call last): File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/concurrent/futures/process.py", line 226, in _process_worker initializer(initargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/job_tools.py", line 270, in worker_initializer _worker_ctx = init_func(init_args) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/core_tools.py", line 155, in _init_binary_worker worker_ctx['recording'] = load_extractor(recording) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 819, in load_extractor return BaseExtractor.from_dict(file_or_folder_or_dict, base_folder=base_folder) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 340, in from_dict extractor = _load_extractor_from_dict(d) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 750, in _load_extractor_from_dict kwargs[k] = _load_extractor_from_dict(v) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 750, in _load_extractor_from_dict kwargs[k] = _load_extractor_from_dict(v) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 750, in _load_extractor_from_dict kwargs[k] = _load_extractor_from_dict(v) [Previous line repeated 2 more times] File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py", line 767, in _load_extractor_from_dict extractor = cls(*kwargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/extractors/nwbextractors.py", line 82, in init self._nwbfile = self.io.read() File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 498, in read return call_docval_func(super().read, kwargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/utils.py", line 424, in call_docval_func return func(fargs, fkwargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/utils.py", line 583, in func_call return func(args[0], pargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/io.py", line 37, in read f_builder = self.read_builder() File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/utils.py", line 583, in func_call return func(args[0], **pargs) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 515, in read_builder f_builder = self.read_group(self.file, ROOT_NAME, ignore=ignore) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 656, in read_group builder = read_method(sub_h5obj) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 637, in __read_group builder = self.read_group(target_obj, builder_name, ignore=ignore) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 656, in read_group builder = read_method(sub_h5obj) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 670, in read_dataset "attributes": self.read_attrs(h5obj), File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 746, in read_attrs ret[k] = self.read_ref(h5obj.file[v]) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 758, in read_ref ret = self.read_group(h5obj) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/hdmf/backends/hdf5/h5tools.py", line 616, in read_group name = str(os.path.basename(h5obj.name)) File "/home/jguidera/anaconda3/envs/nwb_datajoint/lib/python3.8/posixpath.py", line 142, in basename p = os.fspath(p) TypeError: expected str, bytes or os.PathLike object, not NoneType

BrokenProcessPool Traceback (most recent call last) Input In [6], in 9 recording_key = {'nwb_file_name': nwb_file_name2, 10 'sort_group_id': sort_group_id, 11 'sort_interval_name': sort_interval_name, 12 'preproc_params_name': preproc_params_name, 13 'interval_list_name': interval_list_name, 14 'team_name': team_name} 15 nd.common.SpikeSortingRecordingSelection.insert1(recording_key, skip_duplicates=True) ---> 16 nd.common.SpikeSortingRecording.populate([(nd.common.SpikeSortingRecordingSelection & recording_key).proj()]) 18 # sortingview 19 nd.common.SortingviewWorkspace.populate([(nd.common.SpikeSortingRecording & recording_key).proj()])

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/datajoint/autopopulate.py:189, in AutoPopulate.populate(self, suppress_errors, return_exception_objects, reserve_jobs, order, limit, max_calls, display_progress, processes, make_kwargs, *restrictions) 187 if processes == 1: 188 for key in tqdm(keys, desc=self.class.name) if display_progress else keys: --> 189 error = self._populate1(key, jobs, **populate_kwargs) 190 if error is not None: 191 error_list.append(error)

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/datajoint/autopopulate.py:237, in AutoPopulate._populate1(self, key, jobs, suppress_errors, return_exception_objects, make_kwargs) 235 self.class._allow_insert = True 236 try: --> 237 make(dict(key), **(make_kwargs or {})) 238 except (KeyboardInterrupt, SystemExit, Exception) as error: 239 try:

File ~/Src/nwb_datajoint/src/nwb_datajoint/common/common_spikesorting.py:324, in SpikeSortingRecording.make(self, key) 322 if os.path.exists(key['recording_path']): 323 shutil.rmtree(key['recording_path']) --> 324 recording = recording.save(folder=key['recording_path'], n_jobs=4, 325 total_memory='5G') 327 key['recordingid'] = 'R'+str(uuid.uuid4())[:8] 329 self.insert1(key)

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py:579, in BaseExtractor.save(self, kwargs) 577 loaded_extractor = self.save_to_memory(kwargs) 578 else: --> 579 loaded_extractor = self.save_to_folder(**kwargs) 580 return loaded_extractor

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/base.py:654, in BaseExtractor.save_to_folder(self, name, folder, dump_ext, verbose, save_kwargs) 648 provenance_file.write_text( 649 json.dumps({'warning': 'the provenace is not dumpable!!!'}), 650 encoding='utf8' 651 ) 653 # save data (done the subclass) --> 654 cached = self._save(folder=folder, verbose=verbose, save_kwargs) 656 self.save_metadata_to_folder(folder) 658 # copy properties/

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/baserecording.py:197, in BaseRecording._save(self, format, save_kwargs) 194 dtype = self.get_dtype() 196 job_kwargs = {k: save_kwargs[k] for k in self._job_keys if k in save_kwargs} --> 197 write_binary_recording(self, file_paths=file_paths, dtype=dtype, job_kwargs) 199 from .binaryrecordingextractor import BinaryRecordingExtractor 200 cached = BinaryRecordingExtractor(file_paths=file_paths, sampling_frequency=self.get_sampling_frequency(), 201 num_chan=self.get_num_channels(), dtype=dtype, 202 t_starts=t_starts, channel_ids=self.get_channel_ids(), time_axis=0, 203 file_offset=0, gain_to_uV=self.get_channel_gains(), 204 offset_to_uV=self.get_channel_offsets())

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/core_tools.py:240, in write_binary_recording(recording, file_paths, dtype, add_file_extension, verbose, byte_offset, job_kwargs) 237 init_args = (recording.to_dict(), rec_memmaps_dict, dtype) 238 executor = ChunkRecordingExecutor(recording, func, init_func, init_args, verbose=verbose, 239 job_name='write_binary_recording', job_kwargs) --> 240 executor.run()

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/site-packages/spikeinterface/core/job_tools.py:254, in ChunkRecordingExecutor.run(self) 252 returns.append(res) 253 else: --> 254 for res in results: 255 pass 257 return returns

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/concurrent/futures/process.py:484, in _chain_from_iterable_of_lists(iterable) 478 def _chain_from_iterable_of_lists(iterable): 479 """ 480 Specialized implementation of itertools.chain.from_iterable. 481 Each item in iterable should be a list. This function is 482 careful not to keep references to yielded objects. 483 """ --> 484 for element in iterable: 485 element.reverse() 486 while element:

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/concurrent/futures/_base.py:619, in Executor.map..result_iterator() 616 while fs: 617 # Careful not to keep a reference to the popped future 618 if timeout is None: --> 619 yield fs.pop().result() 620 else: 621 yield fs.pop().result(end_time - time.monotonic())

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/concurrent/futures/_base.py:444, in Future.result(self, timeout) 442 raise CancelledError() 443 elif self._state == FINISHED: --> 444 return self.__get_result() 445 else: 446 raise TimeoutError()

File ~/anaconda3/envs/nwb_datajoint/lib/python3.8/concurrent/futures/_base.py:389, in Future.__get_result(self) 387 if self._exception: 388 try: --> 389 raise self._exception 390 finally: 391 # Break a reference cycle with the exception in self._exception 392 self = None

BrokenProcessPool: A process in the process pool was terminated abruptly while the future was running or pending

LorenFrankLab / spyglass

Broken process pool with reasonably sized sort interval #153