search

ChimeraPy / Engine

Distributed computing framework for Multimodal data written in Python
https://chimerapy-engine.readthedocs.io
GNU General Public License v3.0
9 stars 0 forks source link

Runtime error in server.py: Dictionary size changed during iteration. #267

Closed umesh-timalsina closed 1 year ago

umesh-timalsina commented 1 year ago 
2023-09-19 11:00:29 [ERROR] chimerapy-engine: Traceback (most recent call last):
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/networking/async_loop_thread.py", line 36, in _wrapper
    result = await func  # type: ignore
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/manager/manager.py", line 336, in async_shutdown
    await self.eventbus.asend(Event("shutdown"))
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 55, in asend
    await self.stream.asend(event)
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/site-packages/aioreactive/subject.py", line 125, in asend
    await obv.asend(value)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 159, in asend
    await self.exec_callable(self._on_asend)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 136, in exec_callable
    await func(*arg, **kwargs)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/manager/http_server_service.py", line 117, in shutdown
    return await self._server.async_shutdown()
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/networking/server.py", line 432, in async_shutdown
    for ws in self.ws_clients.values():
RuntimeError: dictionary changed size during iteration

Exception in thread Thread-1:
Traceback (most recent call last):
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/networking/async_loop_thread.py", line 81, in run
    self._loop.close()
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/site-packages/asyncio_atexit.py", line 116, in _asyncio_atexit_close
    loop.run_until_complete(_run_asyncio_atexits(loop, entry.callbacks))
  File "uvloop/loop.pyx", line 1515, in uvloop.loop.Loop.run_until_complete
Exception in callback Loop._read_from_self
handle: <Handle Loop._read_from_self>
Traceback (most recent call last):
  File "uvloop/cbhandles.pyx", line 66, in uvloop.loop.Handle._run
  File "uvloop/loop.pyx", line 397, in uvloop.loop.Loop._read_from_self
  File "uvloop/loop.pyx", line 402, in uvloop.loop.Loop._invoke_signals
  File "uvloop/loop.pyx", line 377, in uvloop.loop.Loop._ceval_process_signals
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/dashboard_app.py", line 44, in shutdown_on_sigint
    shutdown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/dashboard_app.py", line 39, in shutdown
    teardown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/init_services.py", line 47, in teardown
    manager.shutdown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/services/cluster_service/cluster_manager.py", line 88, in shutdown
    self._manager.shutdown()
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/manager/manager.py", line 488, in shutdown
    return future.result(timeout=config.get("manager.timeout.worker-shutdown"))
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/concurrent/futures/_base.py", line 458, in result
    return self.__get_result()
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/networking/async_loop_thread.py", line 36, in _wrapper
    result = await func  # type: ignore
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/manager/manager.py", line 336, in async_shutdown
    await self.eventbus.asend(Event("shutdown"))
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 55, in asend
    await self.stream.asend(event)
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/site-packages/aioreactive/subject.py", line 125, in asend
    await obv.asend(value)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 159, in asend
    await self.exec_callable(self._on_asend)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/eventbus/eventbus.py", line 136, in exec_callable
    await func(*arg, **kwargs)
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/manager/http_server_service.py", line 117, in shutdown
    return await self._server.async_shutdown()
  File "/home/umesh/isis/oele-isis/Engine/chimerapy/engine/networking/server.py", line 432, in async_shutdown
    for ws in self.ws_clients.values():
RuntimeError: dictionary changed size during iteration
RuntimeError: Event loop stopped before Future completed.
INFO:     Shutting down
INFO:     Waiting for application shutdown.
ERROR:    Traceback (most recent call last):
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/site-packages/starlette/routing.py", line 677, in lifespan
    async with self.lifespan_context(app) as maybe_state:
  File "/home/umesh/micromamba/envs/chimerapy-dev/lib/python3.10/contextlib.py", line 206, in __aexit__
    await anext(self.gen)
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/dashboard_app.py", line 49, in lifespan
    shutdown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/dashboard_app.py", line 39, in shutdown
    teardown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/init_services.py", line 47, in teardown
    manager.shutdown()
  File "/home/umesh/isis/oele-isis/Orchestrator/chimerapy/orchestrator/services/cluster_service/cluster_manager.py", line 88, in shutdown
    self._manager.shutdown()