search

atarashansky / SAMap

SAMap: Mapping single-cell RNA sequencing datasets from evolutionarily distant organisms.
MIT License
64 stars 19 forks source link

save_samap TypeError: can't pickle weakref objects #76

Closed jmmusser closed 2 years ago

jmmusser commented 2 years ago

Hi Alec,

I'm getting an error when I try to save a samap object. Any suggestions?

I ran samap without any errors. I then used this code to try and save it:

samap.utils.save_samap(sm, "adnv.samap")

I get the following error:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in save(self, key, data)
    482             # If key already exists, we will overwrite the file
--> 483             data_name = overloads[key]
    484         except KeyError:

KeyError: ((array(int64, 2d, C), array(float32, 2d, C), array(float32, 2d, C), type(CPUDispatcher(<function alternative_cosine at 0x7f9cb88d3440>)), array(int64, 1d, C), float64), ('x86_64-apple-darwin20.6.0', 'skylake', '+64bit,+adx,+aes,+avx,+avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512er,-avx512f,-avx512ifma,-avx512pf,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vpopcntdq,+bmi,+bmi2,-cldemote,+clflushopt,-clwb,-clzero,+cmov,+cx16,+cx8,-enqcmd,+f16c,+fma,-fma4,+fsgsbase,+fxsr,-gfni,+invpcid,-lwp,+lzcnt,+mmx,+movbe,-movdir64b,-movdiri,-mwaitx,+pclmul,-pconfig,-pku,+popcnt,-prefetchwt1,+prfchw,-ptwrite,-rdpid,+rdrnd,+rdseed,+rtm,+sahf,+sgx,-sha,-shstk,+sse,+sse2,+sse3,+sse4.1,+sse4.2,-sse4a,+ssse3,-tbm,-vaes,-vpclmulqdq,-waitpkg,-wbnoinvd,-xop,+xsave,+xsavec,+xsaveopt,+xsaves'), ('1566b624ec4710ff21277d163ff1d5d780943ac8328dcb5962340daf38986bdc', 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'))

During handling of the above exception, another exception occurred:

TypeError                                 Traceback (most recent call last)
/var/folders/p3/7mpgsyhd70j5f_n4hj3llyq80000gp/T/ipykernel_56666/954101629.py in <module>
----> 1 samap.utils.save_samap(sm, "/Users/jacobmusser/Documents/Research/EMBL/projects_primary/coral_10x/10x_analysis/samap/samap_objects/adnv.samap")

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/samap/utils.py in save_samap(sm, fn)
     34 
     35     with open(fn,'wb') as f:
---> 36         dill.dump(sm,f)
     37 
     38 def load_samap(fn):

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in dump(obj, file, protocol, byref, fmode, recurse, **kwds)
    274     _kwds = kwds.copy()
    275     _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse))
--> 276     Pickler(file, protocol, **_kwds).dump(obj)
    277     return
    278 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in dump(self, obj)
    496             raise PicklingError(msg)
    497         else:
--> 498             StockPickler.dump(self, obj)
    499         stack.clear()  # clear record of 'recursion-sensitive' pickled objects
    500         return

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in dump(self, obj)
    435         if self.proto >= 4:
    436             self.framer.start_framing()
--> 437         self.save(obj)
    438         self.write(STOP)
    439         self.framer.end_framing()

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    547 
    548         # Save the reduce() output and finally memoize the object
--> 549         self.save_reduce(obj=obj, *rv)
    550 
    551     def persistent_id(self, obj):

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    660 
    661         if state is not None:
--> 662             save(state)
    663             write(BUILD)
    664 

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    988             # we only care about session the first pass thru
    989             pickler._session = False
--> 990         StockPickler.save_dict(pickler, obj)
    991         log.info("# D2")
    992     return

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    988             # we only care about session the first pass thru
    989             pickler._session = False
--> 990         StockPickler.save_dict(pickler, obj)
    991         log.info("# D2")
    992     return

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    547 
    548         # Save the reduce() output and finally memoize the object
--> 549         self.save_reduce(obj=obj, *rv)
    550 
    551     def persistent_id(self, obj):

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    660 
    661         if state is not None:
--> 662             save(state)
    663             write(BUILD)
    664 

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    988             # we only care about session the first pass thru
    989             pickler._session = False
--> 990         StockPickler.save_dict(pickler, obj)
    991         log.info("# D2")
    992     return

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    547 
    548         # Save the reduce() output and finally memoize the object
--> 549         self.save_reduce(obj=obj, *rv)
    550 
    551     def persistent_id(self, obj):

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_reduce(self, func, args, state, listitems, dictitems, obj)
    660 
    661         if state is not None:
--> 662             save(state)
    663             write(BUILD)
    664 

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    502         f = self.dispatch.get(t)
    503         if f is not None:
--> 504             f(self, obj) # Call unbound method with explicit self
    505             return
    506 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/dill/_dill.py in save_module_dict(pickler, obj)
    988             # we only care about session the first pass thru
    989             pickler._session = False
--> 990         StockPickler.save_dict(pickler, obj)
    991         log.info("# D2")
    992     return

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save_dict(self, obj)
    857 
    858         self.memoize(obj)
--> 859         self._batch_setitems(obj.items())
    860 
    861     dispatch[dict] = save_dict

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in _batch_setitems(self, items)
    883                 for k, v in tmp:
    884                     save(k)
--> 885                     save(v)
    886                 write(SETITEMS)
    887             elif n:

/opt/anaconda3/envs/SAMap/lib/python3.7/pickle.py in save(self, obj, save_persistent_id)
    522             reduce = getattr(obj, "__reduce_ex__", None)
    523             if reduce is not None:
--> 524                 rv = reduce(self.proto)
    525             else:
    526                 reduce = getattr(obj, "__reduce__", None)

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/pynndescent/pynndescent_.py in __getstate__(self)
    900     def __getstate__(self):
    901         if not hasattr(self, "_search_graph"):
--> 902             self._init_search_graph()
    903         if not hasattr(self, "_search_function"):
    904             if self._is_sparse:

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/pynndescent/pynndescent_.py in _init_search_graph(self)
    999                     self._distance_func,
   1000                     self.rng_state,
-> 1001                     self.diversify_prob,
   1002                 )
   1003             else:

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
    431                     e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
    432             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 433             raise e
    434 
    435     def inspect_llvm(self, signature=None):

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
    364                 argtypes.append(self.typeof_pyval(a))
    365         try:
--> 366             return self.compile(tuple(argtypes))
    367         except errors.ForceLiteralArg as e:
    368             # Received request for compiler re-entry with the list of arguments

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/dispatcher.py in compile(self, sig)
    861                 raise e.bind_fold_arguments(folded)
    862             self.add_overload(cres)
--> 863             self._cache.save_overload(sig, cres)
    864             return cres.entry_point
    865 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in save_overload(self, sig, data)
    665         """
    666         with self._guard_against_spurious_io_errors():
--> 667             self._save_overload(sig, data)
    668 
    669     def _save_overload(self, sig, data):

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in _save_overload(self, sig, data)
    675         key = self._index_key(sig, _get_codegen(data))
    676         data = self._impl.reduce(data)
--> 677         self._cache_file.save(key, data)
    678 
    679     @contextlib.contextmanager

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in save(self, key, data)
    490                     break
    491             overloads[key] = data_name
--> 492             self._save_index(overloads)
    493         self._save_data(data_name, data)
    494 

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in _save_index(self, overloads)
    536     def _save_index(self, overloads):
    537         data = self._source_stamp, overloads
--> 538         data = self._dump(data)
    539         with self._open_for_write(self._index_path) as f:
    540             pickle.dump(self._version, f, protocol=-1)

/opt/anaconda3/envs/SAMap/lib/python3.7/site-packages/numba/core/caching.py in _dump(self, obj)
    564 
    565     def _dump(self, obj):
--> 566         return pickle.dumps(obj, protocol=-1)
    567 
    568     @contextlib.contextmanager

TypeError: can't pickle weakref objects
jmmusser commented 2 years ago

Hi Alec,

I managed to solve this problem by updating all my packages. Not sure what the exact issue was but for anyone else that encounters this problem I would try updating first.

Jake