Parallel output fails on a Linux machine (running on 6 cores). See also updated commandline notebook ... curiously, the issue does not appear consistently: verbose output -v leads to more failures, and number of cores may play a role (it is not clear how many cores are used for testing pipelines on GitHub).
_________________________________________________________________ TestWrap.test_commandline __________________________________________________________________
self = <test_ctwrap.TestWrap testMethod=test_commandline>
def test_commandline(self):
cmd = 'ctwrap'
name = self._module.__name__.split('.')[-1]
yaml = "{}".format(Path(EXAMPLES) / self._yaml)
pars = [name, yaml, '--parallel']
self.maxDiff = None
process = subprocess.Popen([cmd] + pars,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, stderr = process.communicate()
> self.assertEqual(stderr.decode(), '')
E AssertionError: 'Process Process-1:\nTraceback (most recen[518 chars]nt\n' != ''
E - Process Process-1:
E - Traceback (most recent call last):
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E - self.run()
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E - self._target(*self._args, **self._kwargs)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 695, in worker
E - obj._save_metadata(metadata)
E - UnboundLocalError: local variable 'obj' referenced before assignment
tests/test_ctwrap.py:85: AssertionError
_______________________________________________________________ TestIgnition.test_commandline ________________________________________________________________
self = <test_ctwrap.TestIgnition testMethod=test_commandline>
def test_commandline(self):
cmd = 'ctwrap'
name = self._module.__name__.split('.')[-1]
yaml = "{}".format(Path(EXAMPLES) / self._yaml)
pars = [name, yaml, '--parallel']
self.maxDiff = None
process = subprocess.Popen([cmd] + pars,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
_, stderr = process.communicate()
> self.assertEqual(stderr.decode(), '')
E AssertionError: 'Process Process-1:\nTraceback (most recen[3714 chars]')\n' != ''
E - Process Process-1:
E - Traceback (most recent call last):
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E - self.run()
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E - self._target(*self._args, **self._kwargs)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E - obj._save(task=task)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E - with h5py.File(filename, 'r') as hdf:
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E - swmr=swmr)
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E - fid = h5f.open(name, flags, fapl=fapl)
E - File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E - File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E - File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
E - Process Process-3:
E - Traceback (most recent call last):
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E - self.run()
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E - self._target(*self._args, **self._kwargs)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E - obj._save(task=task)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E - with h5py.File(filename, 'r') as hdf:
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E - swmr=swmr)
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E - fid = h5f.open(name, flags, fapl=fapl)
E - File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E - File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E - File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
E - Process Process-2:
E - Traceback (most recent call last):
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 297, in _bootstrap
E - self.run()
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/multiprocessing/process.py", line 99, in run
E - self._target(*self._args, **self._kwargs)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 687, in worker
E - obj._save(task=task)
E - File "/work/GitHub/ctwrap/ctwrap/simulation.py", line 164, in _save
E - with h5py.File(filename, 'r') as hdf:
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 427, in __init__
E - swmr=swmr)
E - File "/home/ischoegl/.pyenv/versions/miniconda3-latest/envs/ctwrap/lib/python3.7/site-packages/h5py/_hl/files.py", line 190, in make_fid
E - fid = h5f.open(name, flags, fapl=fapl)
E - File "h5py/_objects.pyx", line 54, in h5py._objects.with_phil.wrapper
E - File "h5py/_objects.pyx", line 55, in h5py._objects.with_phil.wrapper
E - File "h5py/h5f.pyx", line 96, in h5py.h5f.open
E - OSError: Unable to open file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
tests/test_ctwrap.py:85: AssertionError
ischoegl
commented
3 years ago
Parallel output fails on a Linux machine (running on 6 cores). See also updated commandline notebook ... curiously, the issue does not appear consistently: verbose output
-vleads to more failures, and number of cores may play a role (it is not clear how many cores are used for testing pipelines on GitHub).