TypeError: full_like() got an unexpected keyword argument 'shape' when running de.test.wald()

[niklaslang]

Hi David,

After using diffxpy for quite some time now, I am using it on a new machine now and I'm running into the following the error when running diffxpy's de.test.wald(): de.test.wald(data = adata_raw.X, formula_loc = '~ 1 + treatment', factor_loc_totest = 'treatment', gene_names = adata_raw.var_names, sample_description = adata_raw.obs, noise_model = 'nb', size_factors = 'n_counts_scaled')

TypeError: full_like() got an unexpected keyword argument 'shape'

I'm running diffxpy==0.7.4 and batchglm==0.7.4 as well as scanpy==1.8.1 anndata==0.7.6 umap==0.5.1 numpy==1.20.3 scipy==1.7.1 pandas==1.3.3 scikit-learn==1.0.1 statsmodels==0.12.2 pynndescent==0.5.4.

Any help is greatly appreciated. Thank you so much!

[davidsebfischer]

Hi @niklaslang thanks for the issue, could you post a full or slightly longer error trace, please?

[niklaslang]

Thanks for getting back to me so quickly! Here's the full trace:

> ---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/tmp/ipykernel_2146984/1501185839.py in <module>
     22         ## step 3: perform DGE testing
     23         print('Started Diffxpy Calculation:\t%s' %sct.util.timestamp())
---> 24         de_test = de.test.wald(data = adata_raw.X,
     25                                formula_loc = '~ 1 + treatment',
     26                                factor_loc_totest = 'treatment',

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/diffxpy/testing/tests.py in wald(data, factor_loc_totest, coef_to_test, formula_loc, formula_scale, as_numeric, init_a, init_b, gene_names, sample_description, dmat_loc, dmat_scale, constraints_loc, constraints_scale, noise_model, size_factors, batch_size, backend, train_args, training_strategy, quick_scale, dtype, **kwargs)
    715 
    716     # Fit model.
--> 717     model = _fit(
    718         noise_model=noise_model,
    719         data=data,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/diffxpy/testing/tests.py in _fit(noise_model, data, design_loc, design_scale, design_loc_names, design_scale_names, constraints_loc, constraints_scale, init_model, init_a, init_b, gene_names, size_factors, batch_size, backend, training_strategy, quick_scale, train_args, close_session, dtype)
    220         raise ValueError('backend="%s" not recognized.' % backend)
    221 
--> 222     estim = Estimator(
    223         input_data=input_data,
    224         init_a=init_a,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/train/numpy/glm_nb/estimator.py in __init__(self, input_data, init_a, init_b, batch_size, quick_scale, dtype, **kwargs)
     57         :param dtype: Numerical precision.
     58         """
---> 59         init_a, init_b, train_loc, train_scale = init_par(
     60             input_data=input_data,
     61             init_a=init_a,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/models/glm_nb/utils.py in init_par(input_data, init_a, init_b, init_model)
    118 
    119             if init_a.lower() == "closed_form":
--> 120                 groupwise_means, init_a, rmsd_a = closedform_nb_glm_logmu(
    121                     x=input_data.x,
    122                     design_loc=input_data.design_loc,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/models/glm_nb/utils.py in closedform_nb_glm_logmu(x, design_loc, constraints_loc, size_factors, link_fn, inv_link_fn)
     28     :return: tuple: (groupwise_means, mu, rmsd)
     29     """
---> 30     return closedform_glm_mean(
     31         x=x,
     32         dmat=design_loc,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/models/base_glm/utils.py in closedform_glm_mean(x, dmat, constraints, size_factors, link_fn, inv_link_fn)
    116             return link_fn(groupwise_means)
    117 
--> 118     linker_groupwise_means, mu, rmsd, rank, s = groupwise_solve_lm(
    119         dmat=dmat,
    120         apply_fun=apply_fun,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/utils/linalg.py in groupwise_solve_lm(dmat, apply_fun, constraints)
     91     # Get group-wise means in linker space based on group assignments
     92     # based on unique rows of design matrix:
---> 93     params = apply_fun(inverse_idx)
     94 
     95     # Use least-squares solver to compute model parameterization

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/batchglm/models/base_glm/utils.py in apply_fun(grouping)
    107 
    108     def apply_fun(grouping):
--> 109         groupwise_means = np.asarray(np.vstack([
    110             np.mean(x[np.where(grouping == g)[0], :], axis=0)
    111             for g in np.unique(grouping)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/numpy/core/_asarray.py in asarray(a, dtype, order, like)
    100         return _asarray_with_like(a, dtype=dtype, order=order, like=like)
    101 
--> 102     return array(a, dtype, copy=False, order=order)
    103 
    104 

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/array/core.py in __array__(self, dtype, **kwargs)
   1539 
   1540     def __array__(self, dtype=None, **kwargs):
-> 1541         x = self.compute()
   1542         if dtype and x.dtype != dtype:
   1543             x = x.astype(dtype)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/base.py in compute(self, **kwargs)
    286         dask.base.compute
    287         """
--> 288         (result,) = compute(self, traverse=False, **kwargs)
    289         return result
    290 

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/base.py in compute(traverse, optimize_graph, scheduler, get, *args, **kwargs)
    569         postcomputes.append(x.__dask_postcompute__())
    570 
--> 571     results = schedule(dsk, keys, **kwargs)
    572     return repack([f(r, *a) for r, (f, a) in zip(results, postcomputes)])
    573 

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/threaded.py in get(dsk, result, cache, num_workers, pool, **kwargs)
     77             pool = MultiprocessingPoolExecutor(pool)
     78 
---> 79     results = get_async(
     80         pool.submit,
     81         pool._max_workers,

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/local.py in get_async(submit, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, chunksize, **kwargs)
    505                             _execute_task(task, data)  # Re-execute locally
    506                         else:
--> 507                             raise_exception(exc, tb)
    508                     res, worker_id = loads(res_info)
    509                     state["cache"][key] = res

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/local.py in reraise(exc, tb)
    313     if exc.__traceback__ is not tb:
    314         raise exc.with_traceback(tb)
--> 315     raise exc
    316 
    317 

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    218     try:
    219         task, data = loads(task_info)
--> 220         result = _execute_task(task, data)
    221         id = get_id()
    222         result = dumps((result, id))

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in <genexpr>(.0)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in <genexpr>(.0)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/optimization.py in __call__(self, *args)
    967         if not len(args) == len(self.inkeys):
    968             raise ValueError("Expected %d args, got %d" % (len(self.inkeys), len(args)))
--> 969         return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
    970 
    971     def __reduce__(self):

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in get(dsk, out, cache)
    147     for key in toposort(dsk):
    148         task = dsk[key]
--> 149         result = _execute_task(task, cache)
    150         cache[key] = result
    151     result = _execute_task(out, cache)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/core.py in _execute_task(arg, cache, dsk)
    117         # temporaries by their reference count and can execute certain
    118         # operations in-place.
--> 119         return func(*(_execute_task(a, cache) for a in args))
    120     elif not ishashable(arg):
    121         return arg

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/utils.py in apply(func, args, kwargs)
     35 def apply(func, args, kwargs=None):
     36     if kwargs:
---> 37         return func(*args, **kwargs)
     38     else:
     39         return func(*args)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/array/reductions.py in mean_chunk(x, sum, numel, dtype, computing_meta, **kwargs)
    584     if computing_meta:
    585         return x
--> 586     n = numel(x, dtype=dtype, **kwargs)
    587 
    588     total = sum(x, dtype=dtype, **kwargs)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/dask/array/reductions.py in numel(x, **kwargs)
    571     else:
    572         new_shape = tuple(shape[dim] for dim in range(len(shape)) if dim not in axis)
--> 573     return np.full_like(x, prod, shape=new_shape, dtype=dtype)
    574 
    575 

<__array_function__ internals> in full_like(*args, **kwargs)

~/miniconda3/envs/niche_fibrosis_env_diffxpy/lib/python3.8/site-packages/sparse/_sparse_array.py in __array_function__(self, func, types, args, kwargs)
    239             pass
    240         else:
--> 241             return sparse_func(*args, **kwargs)
    242 
    243         try:

TypeError: full_like() got an unexpected keyword argument 'shape'

[Zethson]

@davidsebfischer this is an environment error that we cannot figure out how to fix. The very same notebook works with my existing environment, but crashes when @niklaslang and myself create a new environment with only scanpy, diffxpy, sparse==0.9.1, jupyterlab.

Dirty fix: @niklaslang is now using my environment :joy:

Kind of confusing.

[davidsebfischer]

Thanks @Zethson I would guess it s about dask vs sparse version then looking at the error trace.

[Zethson]

Yeah, that's possible. @niklaslang could try dask==2021.4.0. This was reported to work with sparse 0.9.1. See https://github.com/theislab/diffxpy/issues/194

The latest dask release is dask 2021.11.2. Would fit because it was released on the 19th of November this year.