Back in h5py 3.0 the default h5py.File mode was changed from "Read/write if exists, create otherwise" to
"Readonly, file must exist" and while PR https://github.com/NLeSC/noodles/pull/83 was introduced to address these upstream changes, it has yet
to be included in any new noodles release.
With this in mind I'd very much like to up the minimum noodles version for qmflows, as h5py-related
FileNotFoundErrors are starting to pop up on a semi-regular basis.
Example traceback
``` python
Exception in thread Thread-1 (patch):
Traceback (most recent call last):
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/threading.py”, line 1009, in _bootstrap_inner
self.run()
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/threading.py”, line 946, in run
self._target(*self._args, **self._kwargs)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/thread_pool.py”, line 21, in modified_target
return_value = target(*args, **kwargs)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/streams.py”, line 271, in patch
for v in source():
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/streams.py”, line 117, in map
yield from map(self.f, source())
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/run/threading/sqlite3.py”, line 36, in pass_job_stream
status, retrieved_result = db.add_job_to_db(key, job)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/prov/sqlite.py”, line 247, in add_job_to_db
job_msg = self.registry.deep_encode(job)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/serial/registry.py”, line 231, in deep_encode
return deep_map(lambda o: self.encode(o, host), obj)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in deep_map
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in deep_map
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in
return {k: deep_map(f, v) for k, v in result.items()}
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in deep_map
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 106, in
return [deep_map(f, v) for v in result]
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 100, in deep_map
result = f(root)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/serial/registry.py”, line 231, in
return deep_map(lambda o: self.encode(o, host), obj)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/serial/registry.py”, line 187, in encode
result = enc.encode(obj, make_rec)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/serial/numpy.py”, line 83, in encode
f = h5py.File(self.filename)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/h5py/_hl/files.py”, line 507, in __init__
fid = make_fid(name, mode, userblock_size, fapl, fcpl, swmr=swmr)
File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/h5py/_hl/files.py”, line 220, in make_fid
fid = h5f.open(name, flags, fapl=fapl)
File “h5py/_objects.pyx”, line 54, in h5py._objects.with_phil.wrapper
File “h5py/_objects.pyx”, line 55, in h5py._objects.with_phil.wrapper
File “h5py/h5f.pyx”, line 106, in h5py.h5f.open
FileNotFoundError: [Errno 2] Unable to open file (unable to open file: name = ‘cache.hdf5’, errno = 2, error message = ‘No such file or directory’, flags = 0, o_flags = 0)
```
Back in h5py 3.0 the default
h5py.File
mode was changed from "Read/write if exists, create otherwise" to "Readonly, file must exist" and while PR https://github.com/NLeSC/noodles/pull/83 was introduced to address these upstream changes, it has yet to be included in any new noodles release.With this in mind I'd very much like to up the minimum noodles version for qmflows, as h5py-related
FileNotFoundError
s are starting to pop up on a semi-regular basis.Example traceback
``` python Exception in thread Thread-1 (patch): Traceback (most recent call last): File “/home/v13/miniconda3/envs/CAT/lib/python3.10/threading.py”, line 1009, in _bootstrap_inner self.run() File “/home/v13/miniconda3/envs/CAT/lib/python3.10/threading.py”, line 946, in run self._target(*self._args, **self._kwargs) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/thread_pool.py”, line 21, in modified_target return_value = target(*args, **kwargs) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/streams.py”, line 271, in patch for v in source(): File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/streams.py”, line 117, in map yield from map(self.f, source()) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/run/threading/sqlite3.py”, line 36, in pass_job_stream status, retrieved_result = db.add_job_to_db(key, job) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/prov/sqlite.py”, line 247, in add_job_to_db job_msg = self.registry.deep_encode(job) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/serial/registry.py”, line 231, in deep_encode return deep_map(lambda o: self.encode(o, host), obj) File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in deep_map return {k: deep_map(f, v) for k, v in result.items()} File “/home/v13/miniconda3/envs/CAT/lib/python3.10/site-packages/noodles/lib/utility.py”, line 103, in