Closed MTCam closed 2 years ago
strace
, could you put the ~100 lines before the hang in a gist?py-bt-full
from this page works on the machine shop.
- If you ^C at the point of the hang, what happens? Here are a few of those:
_pt_kernel: check array access within bounds: completed (0.54s wall 1.00x CPU)
_pt_kernel: schedule: started 3s ago
_pt_kernel: schedule: completed (3.77s wall 1.00x CPU)
_pt_kernel: generate code: completed (2.60s wall 1.00x CPU)
_pt_kernel: check array access within bounds: completed (0.38s wall 1.00x CPU)
_pt_kernel: check array access within bounds: completed (0.37s wall 1.00x CPU)
^C
Program interrupted. (Use 'cont' to resume).
> /Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py(425)__eq__()
424 return True
--> 425 return (
426 isinstance(other, type(self))
ipdb> c
^C
Program interrupted. (Use 'cont' to resume).
> /Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py(427)<genexpr>()
426 isinstance(other, type(self))
--> 427 and all(
428 getattr(self, field) == getattr(other, field)
ipdb> c
^C
Program interrupted. (Use 'cont' to resume).
--Call--
> /Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py(422)__eq__()
421
--> 422 def __eq__(self, other: Any) -> bool:
423 if self is other:
ipdb>
-----------
^CTraceback (most recent call last):
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/arraycontext/arraycontext/container/traversal.py", line 393, in rec
iterable = serialize_container(_ary)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/functools.py", line 875, in wrapper
return dispatch(args[0].__class__)(*args, **kw)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/arraycontext/arraycontext/container/__init__.py", line 138, in serialize_container
raise TypeError(f"'{type(ary).__name__}' cannot be serialized as a container")
TypeError: 'IndexLambda' cannot be serialized as a container
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/runpy.py", line 194, in _run_module_as_main
return _run_code(code, main_globals, None,
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/site-packages/mpi4py/__main__.py", line 7, in <module>
main()
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/site-packages/mpi4py/run.py", line 198, in main
run_command_line(args)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/site-packages/mpi4py/run.py", line 47, in run_command_line
run_path(sys.argv[0], run_name='__main__')
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/runpy.py", line 265, in run_path
return _run_module_code(code, init_globals, run_name,
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/runpy.py", line 97, in _run_module_code
_run_code(code, mod_globals, init_globals,
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/miniforge3/envs/mirge.x.epsilon/lib/python3.8/runpy.py", line 87, in _run_code
exec(code, run_globals)
File "./autoignition-mpi.py", line 534, in <module>
main(use_logmgr=args.log, use_leap=args.leap, use_profiling=args.profiling,
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/mirgecom/mirgecom/mpi.py", line 154, in wrapped_func
func(*args, **kwargs)
File "./autoignition-mpi.py", line 477, in main
advance_state(rhs=my_rhs, timestepper=timestepper,
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/mirgecom/mirgecom/steppers.py", line 354, in advance_state
_advance_state_stepper_func(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/mirgecom/mirgecom/steppers.py", line 132, in _advance_state_stepper_func
state, dt = post_step_callback(state=state, step=istep, t=t, dt=dt)
File "./autoignition-mpi.py", line 464, in my_post_step
logmgr.tick_after()
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/logpyle/logpyle/__init__.py", line 824, in tick_after
self._gather_for_descriptor(gd)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/logpyle/logpyle/__init__.py", line 772, in _gather_for_descriptor
q_value = gd.quantity()
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/mirgecom/mirgecom/logging_quantities.py", line 285, in __call__
return self._discr_reduction(quantity)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/grudge/grudge/eager.py", line 87, in nodal_max
return op.nodal_max(self, dd, vec)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/grudge/grudge/reductions.py", line 226, in nodal_max
return comm.allreduce(nodal_max_loc(dcoll, dd, vec), op=MPI.MAX)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/grudge/grudge/reductions.py", line 248, in nodal_max_loc
return reduce(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/grudge/grudge/reductions.py", line 251, in <lambda>
actx.to_numpy(actx.np.max(grp_ary))[()]),
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/meshmode/meshmode/array_context.py", line 566, in max
frozen_result = rec_map_reduce_array_container(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/arraycontext/arraycontext/container/traversal.py", line 401, in rec_map_reduce_array_container
return rec(ary)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/arraycontext/arraycontext/container/traversal.py", line 395, in rec
return map_func(_ary)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/meshmode/meshmode/array_context.py", line 568, in <lambda>
lambda ary: cl_array.max(self._array_context.freeze(ary),
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/arraycontext/arraycontext/impl/pytato/__init__.py", line 150, in freeze
pt_prg = self._freeze_prg_cache[normalized_expr]
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 715, in __eq__
and self._data == other._data)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 428, in <genexpr>
getattr(self, field) == getattr(other, field)
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in __eq__
and all(
File "/Users/mtcampbe/CEESD/devel/epsilon/emirge/pytato/pytato/array.py", line 427, in <genexpr>
and all(
KeyboardInterrupt
Abort(1) on node 0 (rank 0 in comm 0): application called MPI_Abort(MPI_COMM_WORLD, 1) - process 0
(
This appears to be processing logpyle's reductions.
- If you run this under
strace
, could you put the ~100 lines before the hang in a gist?
https://gist.github.com/MTCam/e71ed21a686be85415d806155bd84ded
This issue is observed on Mac x86, and Lassen GPUs. In both cases, several successful runs are followed by a hang, and all subsequent runs will hang.
To reproduce:
parallel-lazy
branch of MIRGE-Com (make sure to pick up all the required sub-package branches)thermo-test-lazy
mirgecom/thermochemistry.py
and setnum_iters > 2
inget_temperature
python -m mpi4py autoignition-mpi.py --lazy
It may work several times in a row - but will eventually hang and never work again.