search

dbbs-lab / bsb-core

The Brain Scaffold Builder
https://bsb.readthedocs.io
GNU General Public License v3.0
22 stars 16 forks source link

MPI: Exception during placement #682

Closed drodarie closed 1 year ago

drodarie commented 1 year ago

Tried to use multithreading with attached configuration: config_ann.zip The network can be compiled in mono-thread but with the following command: mpirun -n 2 python config_ann.py I got the following exception during placement:

Traceback (most recent call last):
  File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
    self.run()
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/zwembad/pool.py", line 32, in run
    MPI.COMM_WORLD.send((self._task, (self._args, self._kwargs)), dest=self._worker)
  File "mpi4py/MPI/Comm.pyx", line 1406, in mpi4py.MPI.Comm.send
  File "mpi4py/MPI/msgpickle.pxi", line 211, in mpi4py.MPI.PyMPI_send
  File "mpi4py/MPI/msgpickle.pxi", line 145, in mpi4py.MPI.pickle_dump
  File "mpi4py/MPI/msgpickle.pxi", line 133, in mpi4py.MPI.cdumps
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 364, in dumps
    dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio)
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 336, in dump
    Pickler(file, protocol, **_kwds).dump(obj)
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 620, in dump
    StockPickler.dump(self, obj)
  File "/usr/lib/python3.8/pickle.py", line 487, in dump
    self.save(obj)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 901, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1963, in save_function
    _save_with_postproc(pickler, (_create_function, (
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1140, in _save_with_postproc
    pickler.save_reduce(*reduction, obj=obj)
  File "/usr/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1251, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/lib/python3.8/pickle.py", line 1002, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1963, in save_function
    _save_with_postproc(pickler, (_create_function, (
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1140, in _save_with_postproc
    pickler.save_reduce(*reduction, obj=obj)
  File "/usr/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/usr/lib/python3.8/pickle.py", line 886, in save_tuple
    save(element)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1251, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/lib/python3.8/pickle.py", line 1002, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1963, in save_function
    _save_with_postproc(pickler, (_create_function, (
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1154, in _save_with_postproc
    pickler._batch_setitems(iter(source.items()))
  File "/usr/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1765, in save_module
    pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
  File "/usr/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1251, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1765, in save_module
    pickler.save_reduce(_import_module, (obj.__name__,), obj=obj,
  File "/usr/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1251, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 603, in save
    self.save_reduce(obj=obj, *rv)
  File "/usr/lib/python3.8/pickle.py", line 717, in save_reduce
    save(state)
  File "/usr/lib/python3.8/pickle.py", line 560, in save
    f(self, obj)  # Call unbound method with explicit self
  File "/home/toromis/Workspace/dbbs/venv/lib/python3.8/site-packages/dill/_dill.py", line 1251, in save_module_dict
    StockPickler.save_dict(pickler, obj)
  File "/usr/lib/python3.8/pickle.py", line 971, in save_dict
    self._batch_setitems(obj.items())
  File "/usr/lib/python3.8/pickle.py", line 997, in _batch_setitems
    save(v)
  File "/usr/lib/python3.8/pickle.py", line 578, in save
    rv = reduce(self.proto)
TypeError: cannot pickle 'mpi4py.MPI.Intracomm' object
Helveg commented 1 year ago

I fail to reproduce this, the script runs fine.