search

aertslab / pycisTopic

pycisTopic is a Python module to simultaneously identify cell states and cis-regulatory topics from single cell epigenomics data.
Other
57 stars 12 forks source link

run_umap errors [BUG] #92

Open jklvrt opened 1 year ago

jklvrt commented 1 year ago

hi, looking to transfer all my multiome analyses to this comprehensive package :) however, running into an issue and can't think of potential solutions... and thanks in advance for troubleshooting help.

run_umap gives an error from the following command:

from pycisTopic.clust_vis import * run_umap(cistopic_obj, target = 'cell', scale=True)

error message:

TypingError Traceback (most recent call last) Cell In[90], line 2 1 from pycisTopic.clust_vis import * ----> 2 run_umap(cistopic_obj, target = 'cell', scale=True) 3 plot_metadata(cistopic_obj, reduction_name = 'UMAP', variables = ['celltype'])

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/pycisTopic/clust_vis.py:241, in run_umap(cistopic_obj, target, scale, reduction_name, random_state, selected_topics, selected_features, harmony, rna_components, rna_weight, kwargs) 239 if rna_components is None: 240 reducer = umap.UMAP(random_state=random_state, kwargs) --> 241 embedding = reducer.fit_transform(data_mat) 242 else: 243 atac_topics, rna_components, data_names = input_check(data_mat, rna_components)

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/umap/umap_.py:2772, in UMAP.fit_transform(self, X, y) 2742 def fit_transform(self, X, y=None): 2743 """Fit X into an embedded space and return that transformed 2744 output. 2745 (...) 2770 Local radii of data points in the embedding (log-transformed). 2771 """ -> 2772 self.fit(X, y) 2773 if self.transform_mode == "embedding": 2774 if self.output_dens:

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/umap/umap_.py:2516, in UMAP.fit(self, X, y) 2510 nn_metric = self._input_distance_func 2511 if self.knn_dists is None: 2512 ( 2513 self._knn_indices, 2514 self._knn_dists, 2515 self._knn_search_index, -> 2516 ) = nearest_neighbors( 2517 X[index], 2518 self._n_neighbors, 2519 nn_metric, 2520 self._metric_kwds, 2521 self.angular_rp_forest, 2522 random_state, 2523 self.low_memory, 2524 use_pynndescent=True, 2525 n_jobs=self.n_jobs, 2526 verbose=self.verbose, 2527 ) 2528 else: 2529 self._knn_indices = self.knn_indices

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/umap/umap_.py:328, in nearest_neighbors(X, n_neighbors, metric, metric_kwds, angular, random_state, low_memory, use_pynndescent, n_jobs, verbose) 325 n_trees = min(64, 5 + int(round((X.shape[0]) ** 0.5 / 20.0))) 326 n_iters = max(5, int(round(np.log2(X.shape[0])))) --> 328 knn_search_index = NNDescent( 329 X, 330 n_neighbors=n_neighbors, 331 metric=metric, 332 metric_kwds=metric_kwds, 333 random_state=random_state, 334 n_trees=n_trees, 335 n_iters=n_iters, 336 max_candidates=60, 337 low_memory=low_memory, 338 n_jobs=n_jobs, 339 verbose=verbose, 340 compressed=False, 341 ) 342 knn_indices, knn_dists = knn_search_index.neighbor_graph 344 if verbose:

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/pynndescent/pynndescent_.py:921, in NNDescent.init(self, data, metric, metric_kwds, n_neighbors, n_trees, leaf_size, pruning_degree_multiplier, diversify_prob, n_search_trees, tree_init, init_graph, init_dist, random_state, low_memory, max_candidates, n_iters, delta, n_jobs, compressed, parallel_batch_queries, verbose) 918 if verbose: 919 print(ts(), "NN descent for", str(n_iters), "iterations") --> 921 self._neighbor_graph = nn_descent( 922 self._raw_data, 923 self.n_neighbors, 924 self.rng_state, 925 effective_max_candidates, 926 self._distance_func, 927 self.n_iters, 928 self.delta, 929 low_memory=self.low_memory, 930 rp_tree_init=True, 931 init_graph=_init_graph, 932 leaf_array=leaf_array, 933 verbose=verbose, 934 ) 936 if np.any(self._neighbor_graph[0] < 0): 937 warn( 938 "Failed to correctly find n_neighbors for some samples." 939 " Results may be less than ideal. Try re-running with" 940 " different parameters." 941 )

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/numba/core/dispatcher.py:468, in _DispatcherBase._compile_for_args(self, *args, **kws) 464 msg = (f"{str(e).rstrip()} \n\nThis error may have been caused " 465 f"by the following argument(s):\n{args_str}\n") 466 e.patch_message(msg) --> 468 error_rewrite(e, 'typing') 469 except errors.UnsupportedError as e: 470 # Something unsupported is present in the user code, add help info 471 error_rewrite(e, 'unsupported_error')

File ~/miniconda3/envs/scenicplus/lib/python3.8/site-packages/numba/core/dispatcher.py:409, in _DispatcherBase._compile_for_args..error_rewrite(e, issue_type) 407 raise e 408 else: --> 409 raise e.with_traceback(None)

TypingError: Failed in nopython mode pipeline (step: nopython frontend) Failed in nopython mode pipeline (step: nopython frontend) Untyped global name 'print': Cannot determine Numba type of <class 'function'>

File "miniconda3/envs/scenicplus/lib/python3.8/site-packages/pynndescent/pynndescent_.py", line 252: def nn_descent_internal_low_memory_parallel(

    if verbose:
        print("\t", n + 1, " / ", n_iters)
        ^

During: resolving callee type: type(CPUDispatcher(<function nn_descent_internal_low_memoryparallel at 0x7f401fc72700>)) During: typing of call at /home/jonathan/miniconda3/envs/scenicplus/lib/python3.8/site-packages/pynndescent/pynndescent.py (358)

During: resolving callee type: type(CPUDispatcher(<function nn_descent_internal_low_memoryparallel at 0x7f401fc72700>)) During: typing of call at /home/jonathan/miniconda3/envs/scenicplus/lib/python3.8/site-packages/pynndescent/pynndescent.py (358)

File "miniconda3/envs/scenicplus/lib/python3.8/site-packages/pynndescent/pynndescent_.py", line 358: def nn_descent(

if low_memory:
    nn_descent_internal_low_memory_parallel(
    ^

session info:

pycistarget 1.0.3.dev1+g3fde1ce pycisTopic 1.0.3.dev18+ge563fb6 numba 0.57.1

jklvrt commented 1 year ago

initially solved this error by just exporting the cistopic object, and reimporting it as a new object. now, this solution does not work anymore and i get the same exact errors! any ideas?

SeppeDeWinter commented 1 year ago

Hi @jklvrt

Which versions of umap and pynndescent do you have?


import umap
import pynndescent

umap.__version__
pynndescent.__version__

Maybe updating to the latest version will fix your issue?

Best,

Seppe

mariaZig commented 5 months ago

Hello,

Thanks for the development of this wonderful tool!

I'm facing the exact same error as @jklvrt when trying to run:

run_umap(cistopic_obj, target = 'cell', scale=True)

Is there a fix?

I'm using the following versions of umap, pynndescent, numba and pycistopic:

umap-learn - 0.5.6 pynndescent - 0.5.12 numba - 0.58.1 pycistopic - 1.0a0

All installed in a dedicated conda environment!

Thanks in advance!

xoelmb commented 5 months ago

Hi! First, thanks for this tool. It is amazing. I'm using it as part of the full SCENIC+ pipeline.

Second, same issue here, with library versions:

I should mention that I also force-installed Dask 2024.4.1 previously, which was needed to make the last step in SCENIC+'s pipeline work (related to snakemake).

pycisTopic's tutorial worked fine previously (before I changed that version of Dask), but it is crashing with my dataset (and the changed Dask version), which contains (A) fewer cells (B) from 4 samples. I don't know if it is related.

In the mentioned issue, somebody pointed that we need numba>= 0.57.0, which I have. Another person suggested updating pynndescent. I've installed pynndescent 0.5.12 and...

  1. I tried to reimport the library with importlib, checked that the version was updated, but run_umap crashed again.
  2. I restarted the kernel, reloaded the object from the pickle file and now the problem is gone.

My versions now are

I hope this info helps other people encountering the problem and the devs building the requirements,

My lesson: always restart the kernel. I/You never know what's going on under-the-hood.

Thanks again!

solvi808 commented 2 months ago

Reporting the same error:

I am running the latest version of SCENIC+ with python3.11 .

2024-08-22 20:47:19,339 cisTopic INFO Running UMAP

Traceback (most recent call last):
  File "/home/ulb/iribhm/gwindels/WORKSITE1/phase_2_combined_integration_ONE/SPLUS//LOGS//pycisTopic_run_part3//PIPELINE_MAINSKRIPT.py", line 471, in <module>
    run_umap(cistopic_obj, target  = 'cell', scale=True)
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/pycisTopic/clust_vis.py", line 241, in run_umap
    embedding = reducer.fit_transform(data_mat)
                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/umap/umap_.py", line 2887, in fit_transform
    self.fit(X, y, force_all_finite)
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/umap/umap_.py", line 2608, in fit
    ) = nearest_neighbors(
        ^^^^^^^^^^^^^^^^^^
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/umap/umap_.py", line 329, in nearest_neighbors
    knn_search_index = NNDescent(
                       ^^^^^^^^^^
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/pynndescent/pynndescent_.py", line 931, in __init__
    self._neighbor_graph = nn_descent(
                           ^^^^^^^^^^^
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/numba/core/dispatcher.py", line 468, in _compile_for_args
    error_rewrite(e, 'typing')
  File "/home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/numba/core/dispatcher.py", line 409, in error_rewrite
    raise e.with_traceback(None)
numba.core.errors.TypingError: Failed in nopython mode pipeline (step: nopython frontend)
Failed in nopython mode pipeline (step: nopython frontend)
Untyped global name 'print': Cannot determine Numba type of <class 'function'>

File "../../../VENV/venv_scenicplus/lib/python3.11/site-packages/pynndescent/pynndescent_.py", line 252:
def nn_descent_internal_low_memory_parallel(
    <source elided>
        if verbose:
            print("\t", n + 1, " / ", n_iters)
            ^

During: resolving callee type: type(CPUDispatcher(<function nn_descent_internal_low_memory_parallel at 0x150c7f824e00>))
During: typing of call at /home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/pynndescent/pynndescent_.py (358)

During: resolving callee type: type(CPUDispatcher(<function nn_descent_internal_low_memory_parallel at 0x150c7f824e00>))
During: typing of call at /home/users/g/w/gwindels/VENV/venv_scenicplus/lib/python3.11/site-packages/pynndescent/pynndescent_.py (358)


File "../../../VENV/venv_scenicplus/lib/python3.11/site-packages/pynndescent/pynndescent_.py", line 358:
def nn_descent(
    <source elided>
    if low_memory:
        nn_descent_internal_low_memory_parallel(
        ^