File ~/.conda/envs/ks4/lib/python3.11/site-packages/spikeinterface/sorters/runsorter.py:595, in run_sorter_container(sorter_name, recording, mode, container_image, output_folder, remove_existing_folder, delete_output_folder, verbose, raise_error, with_output, delete_container_files, extra_requirements, installation_mode, spikeinterface_version, spikeinterface_folder_source, **sorter_params)
593 if run_error:
594 if raise_error:
--> 595 raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\n{run_sorter_output}")
596 else:
597 if with_output:
SpikeSortingError: Spike sorting in singularity failed with the following error:
0%| | 0/844 [00:00<?, ?it/s]/home/miniconda3/lib/python3.11/site-packages/kilosort/spikedetect.py:134: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)
B = conv1d(X.unsqueeze(1), W, padding=nt//2)
100%|██████████| 844/844 [03:44<00:00, 3.76it/s]
100%|██████████| 844/844 [03:44<00:00, 3.76it/s]
100%|██████████| 96/96 [02:00<00:00, 1.25s/it]
/home/miniconda3/lib/python3.11/site-packages/kilosort/template_matching.py:78: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:919.)
X = conv1d(X.unsqueeze(1), ops['wTEMP'].unsqueeze(1), padding=ops['nt']//2)
100%|██████████| 844/844 [01:07<00:00, 12.50it/s]
100%|██████████| 96/96 [01:47<00:00, 1.12s/it]
/home/amz/running_data/npx/raw/20240711_az_PANT06/si_preprocess/in_container_python_base/lib/python3.11/site-packages/spikeinterface/core/basesorting.py:264: UserWarning: The registered recording will not be persistent on disk, but only available in memory
warnings.warn("The registered recording will not be persistent on disk, but only available in memory")
/home/amz/.conda/envs/ks4/lib/python3.11/site-packages/spikeinterface/sorters/runsorter.py(595)run_sorter_container()
593 if run_error:
594 if raise_error:
--> 595 raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\n{run_sorter_output}")
596 else:
597 if with_output:
Describe the issue:
Hi,
I tried to run kilosort 4.0.12 through spikeinterface 0.101.0rc0 with singularity image. However, I got the following error:
SpikeSortingError Traceback (most recent call last)
File ~/Documents/git/analysis/analysis/preprocessing/test_ks4.py:119
116 os.chdir(preprocess_folder)
118 # sort spikes
--> 119 sorting = ss.run_sorter(
120 sorter_name='kilosort4',
121 recording=ap_mcd,
122 folder=str(base_folder/"ks4_full"),
123 singularity_image=True,
124 remove_existing_folder=True,
125 #**ks4_params,
126 )
128 # curate
129 sorting = scur.remove_duplicated_spikes(
130 sorting,
131 censored_period_ms=0.3,
132 method="keep_first_iterative",
133 )
File ~/.conda/envs/ks4/lib/python3.11/site-packages/spikeinterface/sorters/runsorter.py:169, in run_sorter(sorter_name, recording, output_folder, remove_existing_folder, delete_output_folder, verbose, raise_error, docker_image, singularity_image, delete_container_files, with_output, sorter_params) 167 else: 168 container_image = singularity_image --> 169 return run_sorter_container( 170 container_image=container_image, 171 mode=mode, 172 common_kwargs, 173 ) 175 return run_sorter_local(**common_kwargs)
File ~/.conda/envs/ks4/lib/python3.11/site-packages/spikeinterface/sorters/runsorter.py:595, in run_sorter_container(sorter_name, recording, mode, container_image, output_folder, remove_existing_folder, delete_output_folder, verbose, raise_error, with_output, delete_container_files, extra_requirements, installation_mode, spikeinterface_version, spikeinterface_folder_source, **sorter_params) 593 if run_error: 594 if raise_error: --> 595 raise SpikeSortingError(f"Spike sorting in {mode} failed with the following error:\n{run_sorter_output}") 596 else: 597 if with_output:
SpikeSortingError: Spike sorting in singularity failed with the following error: 0%| | 0/844 [00:00<?, ?it/s]/home/miniconda3/lib/python3.11/site-packages/kilosort/spikedetect.py:134: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:919.) B = conv1d(X.unsqueeze(1), W, padding=nt//2) 100%|██████████| 844/844 [03:44<00:00, 3.76it/s] 100%|██████████| 844/844 [03:44<00:00, 3.76it/s] 100%|██████████| 96/96 [02:00<00:00, 1.25s/it] /home/miniconda3/lib/python3.11/site-packages/kilosort/template_matching.py:78: UserWarning: Plan failed with a cudnnException: CUDNN_BACKEND_EXECUTION_PLAN_DESCRIPTOR: cudnnFinalize Descriptor Failed cudnn_status: CUDNN_STATUS_NOT_SUPPORTED (Triggered internally at ../aten/src/ATen/native/cudnn/Conv_v8.cpp:919.) X = conv1d(X.unsqueeze(1), ops['wTEMP'].unsqueeze(1), padding=ops['nt']//2) 100%|██████████| 844/844 [01:07<00:00, 12.50it/s] 100%|██████████| 96/96 [01:47<00:00, 1.12s/it] /home/amz/running_data/npx/raw/20240711_az_PANT06/si_preprocess/in_container_python_base/lib/python3.11/site-packages/spikeinterface/core/basesorting.py:264: UserWarning: The registered recording will not be persistent on disk, but only available in memory warnings.warn("The registered recording will not be persistent on disk, but only available in memory")
Could anyone please help? Thanks!
Best, Arthur
Reproduce the bug:
No response
Error message:
No response
Version information:
python: 3.11 kilosort: 4.0.12 OS: linux fedora 39