Depending on precomputed models broke CI

[cwehmeyer]

We need to make sure that circleci runs the notebooks in a special order to prevent issues like this:

OSError: Unable to open file (unable to open file:
    name = 'nb4.pyemma',
    errno = 2,
    error message = 'No such file or directory',
    flags = 0,
    o_flags = 0)

Traceback

``` Python-Traceback NameError: name 'cgflux' is not defined __________________________________ ::Cell 22 ___________________________________ Notebook cell execution failed Cell 22: Cell execution caused an exception Input: feat = pyemma.coordinates.featurizer(pdb) feat.add_distances(feat.select_Heavy()) data = pyemma.coordinates.load(files, features=feat) tica = pyemma.load('nb4.pyemma', model_name='ala2tica_tica') tica_output = tica.transform(data) tica_concatenated = np.concatenate(tica_output) msm = pyemma.load('nb4.pyemma', model_name='ala2tica_msm') bayesian_msm = pyemma.load('nb4.pyemma', model_name='ala2tica_bayesian_msm') cluster = pyemma.load('nb4.pyemma', model_name='ala2tica_cluster') dtrajs_concatenated = np.concatenate(cluster.dtrajs) Traceback: --------------------------------------------------------------------------- OSError Traceback (most recent call last) in () 3 data = pyemma.coordinates.load(files, features=feat) 4 ----> 5 tica = pyemma.load('nb4.pyemma', model_name='ala2tica_tica') 6 tica_output = tica.transform(data) 7 tica_concatenated = np.concatenate(tica_output) /opt/conda/conda-bld/pyemma_tutorials_1531145192023/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages/pyemma/__init__.py in load(filename, model_name) 56 """ 57 from ._base.serialization.serialization import SerializableMixIn ---> 58 return SerializableMixIn.load(file_name=filename, model_name=model_name) 59 60 /opt/conda/conda-bld/pyemma_tutorials_1531145192023/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages/pyemma/_base/serialization/serialization.py in load(cls, file_name, model_name) 275 raise NotImplementedError('This feature is only available on Python3. Consider upgrading.') 276 from .h5file import H5File --> 277 with H5File(file_name, model_name=model_name, mode='r') as f: 278 return f.model 279 /opt/conda/conda-bld/pyemma_tutorials_1531145192023/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages/pyemma/_base/serialization/h5file.py in __init__(self, file_name, model_name, mode) 39 def __init__(self, file_name, model_name=None, mode='r'): 40 import h5py ---> 41 self._file = h5py.File(file_name, mode=mode) 42 self._parent = self._file.require_group('pyemma') 43 self._current_model_group = model_name /opt/conda/conda-bld/pyemma_tutorials_1531145192023/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages/h5py/_hl/files.py in __init__(self, name, mode, driver, libver, userblock_size, swmr, **kwds) 310 with phil: 311 fapl = make_fapl(driver, libver, **kwds) --> 312 fid = make_fid(name, mode, userblock_size, fapl, swmr=swmr) 313 314 if swmr_support: /opt/conda/conda-bld/pyemma_tutorials_1531145192023/_test_env_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_placehold_pla/lib/python3.6/site-packages/h5py/_hl/files.py in make_fid(name, mode, userblock_size, fapl, fcpl, swmr) 140 if swmr and swmr_support: 141 flags |= h5f.ACC_SWMR_READ --> 142 fid = h5f.open(name, flags, fapl=fapl) 143 elif mode == 'r+': 144 fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) h5py/_objects.pyx in h5py._objects.with_phil.wrapper() h5py/_objects.pyx in h5py._objects.with_phil.wrapper() h5py/h5f.pyx in h5py.h5f.open() OSError: Unable to open file (unable to open file: name = 'nb4.pyemma', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0) __________________________________ ::Cell 24 ___________________________________ Notebook cell execution failed Cell 24: Cell execution caused an exception Input: nstates = 4 msm.pcca(nstates) for i, s in enumerate(msm.metastable_sets): print('π_{} = {:f}'.format(i, msm.pi[s].sum())) fig, axes = plt.subplots(1, 4, figsize=(15, 3)) for i, ax in enumerate(axes.flat): pyemma.plots.plot_contour( *tica_concatenated.T, msm.metastable_distributions[i][dtrajs_concatenated], ax=ax, cmap='afmhot_r', mask=True, method='nearest', cbar_label='metastable distribution {}'.format(i)) ax.set_xlabel('IC 1') axes[0].set_ylabel('IC 2') fig.tight_layout() Traceback: --------------------------------------------------------------------------- NameError Traceback (most recent call last) in () 1 nstates = 4 ----> 2 msm.pcca(nstates) 3 4 for i, s in enumerate(msm.metastable_sets): 5 print('π_{} = {:f}'.format(i, msm.pi[s].sum())) NameError: name 'msm' is not defined __________________________________ ::Cell 26 ___________________________________ Notebook cell execution failed Cell 26: Cell execution caused an exception Input: mfpt = np.zeros((nstates, nstates)) for i in range(nstates): for j in range(nstates): mfpt[i, j] = msm.mfpt( msm.metastable_sets[i], msm.metastable_sets[j]) rate = np.zeros_like(mfpt) nz = mfpt.nonzero() rate[nz] = 1.0 / mfpt[nz] pyemma.plots.plot_network( rate, pos=np.asarray([[0, 0], [4, 0], [2, 4], [6, 4]]), arrow_label_format='%.1f ps', arrow_labels=mfpt, arrow_scale=3.0, size=12); Traceback: --------------------------------------------------------------------------- NameError Traceback (most recent call last) in () 2 for i in range(nstates): 3 for j in range(nstates): ----> 4 mfpt[i, j] = msm.mfpt( 5 msm.metastable_sets[i], 6 msm.metastable_sets[j]) NameError: name 'msm' is not defined __________________________________ ::Cell 27 ___________________________________ Notebook cell execution failed Cell 27: Cell execution caused an exception Input: A = msm.metastable_sets[0] B = msm.metastable_sets[3] flux = #FIXME cg, cgflux = #FIXME highest_membership = msm.metastable_distributions.argmax(1) coarse_state_centers = cluster.clustercenters[msm.active_set[highest_membership]] fig, ax = plt.subplots(figsize=(8, 6)) pyemma.plots.plot_contour( *tica_concatenated.T, flux.committor[dtrajs_concatenated], cmap='brg', ax=ax, mask=True, method='nearest', cbar_label='committor 0 -> 3', alpha=.5); pyemma.plots.plot_flux(cgflux, coarse_state_centers, cgflux.stationary_distribution, ax=ax, show_committor=False, figpadding=0.1) ax.set_aspect('equal') ax.set_xlim(tica_concatenated[:, 0].min(), tica_concatenated[:, 0].max()) ax.set_ylim(tica_concatenated[:, 1].min(), tica_concatenated[:, 1].max()) Traceback: File "", line 3 flux = #FIXME ^ SyntaxError: invalid syntax __________________________________ ::Cell 28 ___________________________________ Notebook cell execution failed Cell 28: Cell execution caused an exception Input: A = msm.metastable_sets[0] B = msm.metastable_sets[3] flux = pyemma.msm.tpt(msm, A, B) cg, cgflux = flux.coarse_grain(msm.metastable_sets) highest_membership = msm.metastable_distributions.argmax(1) coarse_state_centers = cluster.clustercenters[msm.active_set[highest_membership]] fig, ax = plt.subplots(figsize=(4, 8)) pyemma.plots.plot_contour( *tica_concatenated.T, flux.committor[dtrajs_concatenated], cmap='brg', ax=ax, mask=True, method='nearest', cbar_label='committor 0 -> 3', alpha=.8, zorder=-1); pyemma.plots.plot_flux(cgflux, coarse_state_centers, cgflux.stationary_distribution, ax=ax, show_committor=False, figpadding=0.2, arrow_label_format='%2.e / ps') ax.set_xlabel('IC 1') ax.set_ylabel('IC 2') ax.set_aspect('equal') ax.set_xlim(tica_concatenated[:, 0].min(), tica_concatenated[:, 0].max()) ax.set_ylim(tica_concatenated[:, 1].min(), tica_concatenated[:, 1].max()) fig.tight_layout() Traceback: --------------------------------------------------------------------------- NameError Traceback (most recent call last) in () ----> 1 A = msm.metastable_sets[0] 2 B = msm.metastable_sets[3] 3 flux = pyemma.msm.tpt(msm, A, B) 4 5 cg, cgflux = flux.coarse_grain(msm.metastable_sets) NameError: name 'msm' is not defined ------------------- generated xml file: /root/junit/test.xml ------------------- ====== 24 failed, 35 passed, 8 skipped, 182 deselected in 353.82 seconds ======= Tests failed for pyemma_tutorials-dev-0.tar.bz2 - moving package to /opt/conda/conda-bld/broken WARNING:conda_build.build:Tests failed for pyemma_tutorials-dev-0.tar.bz2 - moving package to /opt/conda/conda-bld/broken removing: pyemma_tutorials-dev-0.tar.bz2 TESTS FAILED: pyemma_tutorials-dev-0.tar.bz2 Exited with code 1 ```

[marscher]

pr is on the way.

[orthonalmatrix]

I'm still having this issue: file = mdshare.fetch('hmm-doublewell-2d-100k.npz', working_directory='data') with np.load(file) as fh: data = fh['trajectory'] works, but: msm = pyemma.load('nb4.pyemma', model_name='doublewell_msm') bayesian_msm = pyemma.load('nb4.pyemma', model_name='doublewell_bayesian_msm') cluster = pyemma.load('nb4.pyemma', model_name='doublewell_cluster')

---

FileNotFoundError Traceback (most recent call last)

in ----> 1 msm = pyemma.load('nb4.pyemma', model_name='doublewell_msm') 2 bayesian_msm = pyemma.load('nb4.pyemma', model_name='doublewell_bayesian_msm') 3 cluster = pyemma.load('nb4.pyemma', model_name='doublewell_cluster') ~/bin/anaconda3/envs/pyemma/lib/python3.7/site-packages/pyemma/_base/serialization/__init__.py in load(filename, model_name) 14 def load(filename, model_name='default'): 15 from .serialization import SerializableMixIn ---> 16 return SerializableMixIn.load(file_name=filename, model_name=model_name) 17 18 ~/bin/anaconda3/envs/pyemma/lib/python3.7/site-packages/pyemma/_base/serialization/serialization.py in load(cls, file_name, model_name) 266 """ 267 from .h5file import H5File --> 268 with H5File(file_name, model_name=model_name, mode='r') as f: 269 return f.model 270 ~/bin/anaconda3/envs/pyemma/lib/python3.7/site-packages/pyemma/_base/serialization/h5file.py in __init__(self, file_name, model_name, mode) 39 def __init__(self, file_name, model_name=None, mode='r'): 40 import h5py ---> 41 self._file = h5py.File(file_name, mode=mode) 42 self._parent = self._file.require_group('pyemma') 43 self._current_model_group = model_name ~/bin/anaconda3/envs/pyemma/lib/python3.7/site-packages/h5py/_hl/files.py in __init__(self, name, mode, driver, libver, userblock_size, swmr, rdcc_nslots, rdcc_nbytes, rdcc_w0, track_order, fs_strategy, fs_persist, fs_threshold, **kwds) 445 fapl, fcpl=make_fcpl(track_order=track_order, fs_strategy=fs_strategy, 446 fs_persist=fs_persist, fs_threshold=fs_threshold), --> 447 swmr=swmr) 448 449 if isinstance(libver, tuple): ~/bin/anaconda3/envs/pyemma/lib/python3.7/site-packages/h5py/_hl/files.py in make_fid(name, mode, userblock_size, fapl, fcpl, swmr) 197 if swmr and swmr_support: 198 flags |= h5f.ACC_SWMR_READ --> 199 fid = h5f.open(name, flags, fapl=fapl) 200 elif mode == 'r+': 201 fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) h5py/_objects.pyx in h5py._objects.with_phil.wrapper() h5py/_objects.pyx in h5py._objects.with_phil.wrapper() h5py/h5f.pyx in h5py.h5f.open() FileNotFoundError: [Errno 2] Unable to open file (unable to open file: name = 'nb4.pyemma', errno = 2, error message = 'No such file or directory', flags = 0, o_flags = 0)