pytest: eep0 issues - Githubissues

Emily and I ran into a few issues, mostly related to eep0, when running pytest on tests.py

Summary: ================================================================= short test summary info ================================================================= FAILED tests.py::test_mist_basic - ValueError: eep0 gives nan for all initial guesses! (1.0, 9.5, -0.2) FAILED tests.py::test_closest_eep - NotImplementedError FAILED tests.py::test_spec - ValueError: eep0 gives nan for all initial guesses! (1.0, 9.6, 0.1) FAILED tests.py::test_tracks - tarfile.ReadError: file could not be opened successfully ========================================================= 4 failed, 3 passed, 1 warning in 30.45s =========================================================

With more details below. I didn't run into these issues when installing isochrones on my laptop, so I think it may be related to the architecture or environment of the cluster we're using. We tried to put as many packages into the Conda environment as possible in the Conda create step (everything except emcee, corner, astroquery, and isochrones) and used version 1.0 of pandas. If you happen to see anything below that might point to what's wrong, we'd appreciate it!

color-coded.pdf

============================= test session starts ============================== platform linux -- Python 3.6.11, pytest-6.1.0, py-1.9.0, pluggy-0.13.1 rootdir: /storage/work/r/rxd44/.conda/envs/isonew/lib/python3.6/site-packages/isochrones/tests plugins: flaky-3.7.0 collected 7 items

tests.py FFFF... [100%]

=================================== FAILURES =================================== ___ test_mist_basic ____

self = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae4487f60> mass = 1.0, age = 9.5, feh = -0.2, eep0 = 600, resid_tol = 0.02 method = 'nelder-mead', return_object = False, return_nan = False, kwargs = {} eeps_to_try = []

def get_eep_accurate(
    self,
    mass,
    age,
    feh,
    eep0=300,
    resid_tol=0.02,
    method="nelder-mead",
    return_object=False,
    return_nan=False,
    **kwargs,
):

    eeps_to_try = [min(self.max_eep(mass, feh) - 20, 600), 100, 200]
    while np.isnan(self.mass_age_resid(eep0, mass, age, feh)):
        try:

          eep0 = eeps_to_try.pop()
E IndexError: pop from empty list

../models.py:560: IndexError

During handling of the above exception, another exception occurred:

bands = 'JHK'

def test_mist_basic(bands="JHK"):
    ic = MIST_Isochrone(bands)

  _basic_ic_checks(ic)

tests.py:14:

tests.py:92: in _basic_ic_checks eep = ic.get_eep(1.0, age, feh, accurate=True) ../models.py:509: in get_eep return self.get_eep_accurate(mass, age, feh, **kwargs)

def get_eep_accurate(
    self,
    mass,
    age,
    feh,
    eep0=300,
    resid_tol=0.02,
    method="nelder-mead",
    return_object=False,
    return_nan=False,
    **kwargs,
):

    eeps_to_try = [min(self.max_eep(mass, feh) - 20, 600), 100, 200]
    while np.isnan(self.mass_age_resid(eep0, mass, age, feh)):
        try:
            eep0 = eeps_to_try.pop()
        except IndexError:
            if return_nan:
                return np.nan
            else:

              raise ValueError("eep0 gives nan for all initial guesses! {}".format((mass, age, feh)))
E ValueError: eep0 gives nan for all initial guesses! (1.0, 9.5, -0.2)

../models.py:565: ValueError _ test_closesteep

ic = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828> n = 10000, resid_tol = 0.02

def _check_closest_eep(ic, n=3000, resid_tol=0.02):
    np.random.seed(1234)
    masses = np.random.random(n) * 2.5 + 0.1
    fehs = np.random.random(n) * (ic.maxfeh - ic.minfeh) + ic.minfeh
    ages = np.random.random(n) * (10.0 - ic.minage) + ic.minage
    eeps = [
        ic.get_eep(m, a, f, return_nan=True, resid_tol=resid_tol, accurate=True)
        for m, a, f in zip(masses, ages, fehs)
    ]
    for e, a, f, m in zip(eeps, ages, fehs, masses):
        if not np.isnan(e):
            try:
                assert abs(ic.initial_mass(e, a, f) - m) < (resid_tol * 1.1)  # approx check
            except AssertionError:
                print("{}: {}".format((m, a, f), e))
                raise
            # print (('{:.4f} ' * 5).format(e, a, f, m, mist.initial_mass(e, a, f)))

    # make sure the minmass edge case works.
    for feh in ic.fehs[1:-1]:  # first and last feh of mist doesn't work.
        try:

          assert np.isfinite(
ic.get_eep(ic.minmass + 0.01, 9.0, feh, return_nan=True, resid_tol=resid_tol, accurate=True) ) E AssertionError: assert False E + where False = <ufunc 'isfinite'>(nan) E + where <ufunc 'isfinite'> = np.isfinite E + and nan = <bound method ModelGridInterpolator.get_eep of <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828>>((0.1 + 0.01), 9.0, -3.5, return_nan=True, resid_tol=0.02, accurate=True) E + where <bound method ModelGridInterpolator.get_eep of <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828>> = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828>.get_eep E + and 0.1 = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828>.minmass

tests.py:81: AssertionError

During handling of the above exception, another exception occurred:

n = 10000

def test_closest_eep(n=10000):
    mist = get_ichrone("mist")

  _check_closest_eep(mist, n=n)

tests.py:24:

tests.py:85: in _check_closest_eep ic.get_eep(ic.minmass + 0.01, 9.0, feh, debug=True)

self = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae117f828> mass = 0.11, age = 9.0, feh = -3.5, accurate = False, kwargs = {'debug': True} grid = <isochrones.mist.models.MISTIsochroneGrid object at 0x7f7ae117feb8>

def get_eep(self, mass, age, feh, accurate=False, **kwargs):
    grid = self.model_grid
    if (
        (isinstance(mass, float) or isinstance(mass, int))
        and (isinstance(age, float) or isinstance(age, int))
        and (isinstance(feh, float) or isinstance(feh, int))
    ):
        if accurate:
            return self.get_eep_accurate(mass, age, feh, **kwargs)
        else:
            if grid.eep_replaces == "age":
                return interp_eep(
                    age,
                    feh,
                    mass,
                    grid.fehs,
                    grid.masses,
                    grid.n_masses,
                    grid.age_grid,
                    grid.dt_deep_grid,
                    grid.array_lengths,
                )
            elif grid.eep_replaces == "mass":

              raise NotImplementedError
E NotImplementedError

../models.py:524: NotImplementedError __ test_spec ___

self = <isochrones.mist.isochrone.MIST_Isochrone object at 0x7f7ae3600ac8> mass = 1.0, age = 9.6, feh = 0.1, eep0 = 600, resid_tol = 0.02 method = 'nelder-mead', return_object = False, return_nan = False, kwargs = {} eeps_to_try = []

def get_eep_accurate(
    self,
    mass,
    age,
    feh,
    eep0=300,
    resid_tol=0.02,
    method="nelder-mead",
    return_object=False,
    return_nan=False,
    **kwargs,
):

    eeps_to_try = [min(self.max_eep(mass, feh) - 20, 600), 100, 200]
    while np.isnan(self.mass_age_resid(eep0, mass, age, feh)):
        try:

          eep0 = eeps_to_try.pop()
E IndexError: pop from empty list

../models.py:560: IndexError

During handling of the above exception, another exception occurred:

bands = 'JHK'

def test_spec(bands="JHK"):
    mist = get_ichrone("mist", bands=bands)

  _check_spec(mist)

tests.py:29:

tests.py:124: in _check_spec eep = ic.get_eep(1.0, 9.6, 0.1, accurate=True) ../models.py:509: in get_eep return self.get_eep_accurate(mass, age, feh, **kwargs)

def get_eep_accurate(
    self,
    mass,
    age,
    feh,
    eep0=300,
    resid_tol=0.02,
    method="nelder-mead",
    return_object=False,
    return_nan=False,
    **kwargs,
):

    eeps_to_try = [min(self.max_eep(mass, feh) - 20, 600), 100, 200]
    while np.isnan(self.mass_age_resid(eep0, mass, age, feh)):
        try:
            eep0 = eeps_to_try.pop()
        except IndexError:
            if return_nan:
                return np.nan
            else:

              raise ValueError("eep0 gives nan for all initial guesses! {}".format((mass, age, feh)))
E ValueError: eep0 gives nan for all initial guesses! (1.0, 9.6, 0.1)

../models.py:565: ValueError _ test_tracks __

self = <isochrones.mist.models.MISTEvolutionTrackGrid object at 0x7f7ae1c17048> orig = False

def read_hdf(self, orig=False):
    h5file = self.hdf_filename
    try:
        path = "orig" if orig else "df"

      df = pd.read_hdf(h5file, path)

../grid.py:107:

path_or_buf = '/storage/home/rxd44/.isochrones/mist/tracks/mist_v1.2_vvcrit0.4.h5' key = 'df', mode = 'r', errors = 'strict', where = None, start = None stop = None, columns = None, iterator = False, chunksize = None, kwargs = {} exists = False

def read_hdf(
    path_or_buf,
    key=None,
    mode: str = "r",
    errors: str = "strict",
    where=None,
    start: Optional[int] = None,
    stop: Optional[int] = None,
    columns=None,
    iterator=False,
    chunksize: Optional[int] = None,
    **kwargs,
):
    """
    Read from the store, close it if we opened it.

    Retrieve pandas object stored in file, optionally based on where
    criteria

    Parameters
    ----------
    path_or_buf : str, path object, pandas.HDFStore or file-like object
        Any valid string path is acceptable. The string could be a URL. Valid
        URL schemes include http, ftp, s3, and file. For file URLs, a host is
        expected. A local file could be: ``file://localhost/path/to/table.h5``.

        If you want to pass in a path object, pandas accepts any
        ``os.PathLike``.

        Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.

        By file-like object, we refer to objects with a ``read()`` method,
        such as a file handler (e.g. via builtin ``open`` function)
        or ``StringIO``.

        .. versionadded:: 0.21.0 support for __fspath__ protocol.

    key : object, optional
        The group identifier in the store. Can be omitted if the HDF file
        contains a single pandas object.
    mode : {'r', 'r+', 'a'}, default 'r'
        Mode to use when opening the file. Ignored if path_or_buf is a
        :class:`pandas.HDFStore`. Default is 'r'.
    where : list, optional
        A list of Term (or convertible) objects.
    start : int, optional
        Row number to start selection.
    stop  : int, optional
        Row number to stop selection.
    columns : list, optional
        A list of columns names to return.
    iterator : bool, optional
        Return an iterator object.
    chunksize : int, optional
        Number of rows to include in an iteration when using an iterator.
    errors : str, default 'strict'
        Specifies how encoding and decoding errors are to be handled.
        See the errors argument for :func:`open` for a full list
        of options.
    **kwargs
        Additional keyword arguments passed to HDFStore.

    Returns
    -------
    item : object
        The selected object. Return type depends on the object stored.

    See Also
    --------
    DataFrame.to_hdf : Write a HDF file from a DataFrame.
    HDFStore : Low-level access to HDF files.

    Examples
    --------
    >>> df = pd.DataFrame([[1, 1.0, 'a']], columns=['x', 'y', 'z'])
    >>> df.to_hdf('./store.h5', 'data')
    >>> reread = pd.read_hdf('./store.h5')
    """

    if mode not in ["r", "r+", "a"]:
        raise ValueError(
            f"mode {mode} is not allowed while performing a read. "
            f"Allowed modes are r, r+ and a."
        )
    # grab the scope
    if where is not None:
        where = _ensure_term(where, scope_level=1)

    if isinstance(path_or_buf, HDFStore):
        if not path_or_buf.is_open:
            raise IOError("The HDFStore must be open for reading.")

        store = path_or_buf
        auto_close = False
    else:
        path_or_buf = stringify_path(path_or_buf)
        if not isinstance(path_or_buf, str):
            raise NotImplementedError(
                "Support for generic buffers has not been implemented."
            )
        try:
            exists = os.path.exists(path_or_buf)

        # if filepath is too long
        except (TypeError, ValueError):
            exists = False

        if not exists:

          raise FileNotFoundError(f"File {path_or_buf} does not exist")
E FileNotFoundError: File /storage/home/rxd44/.isochrones/mist/tracks/mist_v1.2_vvcrit0.4.h5 does not exist

../../pandas/io/pytables.py:395: FileNotFoundError

During handling of the above exception, another exception occurred:

bands = 'JHK'

def test_tracks(bands="JHK"):
    mist = get_ichrone("mist", bands=bands, tracks=True)

  _basic_ic_checks_tracks(mist)

tests.py:34:

tests.py:130: in _basic_ic_checks_tracks eep = ic.get_eep(mass, 9.6, feh, accurate=True) ../models.py:509: in get_eep return self.get_eep_accurate(mass, age, feh, **kwargs) ../models.py:558: in get_eep_accurate while np.isnan(self.mass_age_resid(eep0, mass, age, feh)): ../models.py:686: in mass_age_resid age_interp = self.interp_value([mass, eep, feh], ["age"]) ../models.py:400: in interp_value return self.model_grid.interp(pars, props) ../grid.py:136: in interp self._interp = DFInterpolator(self.df, filename=filename, is_full=self.is_full) ../mist/models.py:398: in df self._df = self.read_hdf() ../grid.py:109: in read_hdf df = self.write_hdf(orig=orig) ../grid.py:113: in write_hdf df = self.get_df(orig=orig) ../models.py:114: in get_df df = self.df_all() ../mist/models.py:392: in df_all df = pd.concat([self.df_all_feh_interpolated(feh) for feh in self.fehs]) ../mist/models.py:392: in df = pd.concat([self.df_all_feh_interpolated(feh) for feh in self.fehs]) ../mist/models.py:326: in df_all_feh_interpolated df = self.df_all_feh(feh) ../mist/models.py:310: in df_all_feh df = pd.concat([self.to_df(f) for f in self.get_feh_filenames(feh)]) ../mist/models.py:294: in get_feh_filenames self.extract_tarball(feh=feh) ../grid.py:95: in extract_tarball with tarfile.open(tarball) as tar:

cls = <class 'tarfile.TarFile'> name = '/storage/home/rxd44/.isochrones/mist/tracks/MIST_v1.2_feh_m4.00_afe_p0.0_vvcrit0.4_EEPS.txz' mode = 'r', fileobj = None, bufsize = 10240, kwargs = {} not_compressed = <function TarFile.open..not_compressed at 0x7f7ae1478488> comptype = 'tar' func = <bound method TarFile.taropen of <class 'tarfile.TarFile'>>

@classmethod
def open(cls, name=None, mode="r", fileobj=None, bufsize=RECORDSIZE, **kwargs):
    """Open a tar archive for reading, writing or appending. Return
       an appropriate TarFile class.

       mode:
       'r' or 'r:*' open for reading with transparent compression
       'r:'         open for reading exclusively uncompressed
       'r:gz'       open for reading with gzip compression
       'r:bz2'      open for reading with bzip2 compression
       'r:xz'       open for reading with lzma compression
       'a' or 'a:'  open for appending, creating the file if necessary
       'w' or 'w:'  open for writing without compression
       'w:gz'       open for writing with gzip compression
       'w:bz2'      open for writing with bzip2 compression
       'w:xz'       open for writing with lzma compression

       'x' or 'x:'  create a tarfile exclusively without compression, raise
                    an exception if the file is already created
       'x:gz'       create a gzip compressed tarfile, raise an exception
                    if the file is already created
       'x:bz2'      create a bzip2 compressed tarfile, raise an exception
                    if the file is already created
       'x:xz'       create an lzma compressed tarfile, raise an exception
                    if the file is already created

       'r|*'        open a stream of tar blocks with transparent compression
       'r|'         open an uncompressed stream of tar blocks for reading
       'r|gz'       open a gzip compressed stream of tar blocks
       'r|bz2'      open a bzip2 compressed stream of tar blocks
       'r|xz'       open an lzma compressed stream of tar blocks
       'w|'         open an uncompressed stream for writing
       'w|gz'       open a gzip compressed stream for writing
       'w|bz2'      open a bzip2 compressed stream for writing
       'w|xz'       open an lzma compressed stream for writing
    """

    if not name and not fileobj:
        raise ValueError("nothing to open")

    if mode in ("r", "r:*"):
        # Find out which *open() is appropriate for opening the file.
        def not_compressed(comptype):
            return cls.OPEN_METH[comptype] == 'taropen'
        for comptype in sorted(cls.OPEN_METH, key=not_compressed):
            func = getattr(cls, cls.OPEN_METH[comptype])
            if fileobj is not None:
                saved_pos = fileobj.tell()
            try:
                return func(name, "r", fileobj, **kwargs)
            except (ReadError, CompressionError):
                if fileobj is not None:
                    fileobj.seek(saved_pos)
                continue

      raise ReadError("file could not be opened successfully")
E tarfile.ReadError: file could not be opened successfully

/storage/home/rxd44/.conda/envs/isonew/lib/python3.6/tarfile.py:1576: ReadError =============================== warnings summary =============================== tests.py::test_mist_basic /storage/home/rxd44/.conda/envs/isonew/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject return f(*args, **kwds)

-- Docs: https://docs.pytest.org/en/stable/warnings.html =========================== short test summary info ============================ FAILED tests.py::test_mist_basic - ValueError: eep0 gives nan for all initial... FAILED tests.py::test_closest_eep - NotImplementedError FAILED tests.py::test_spec - ValueError: eep0 gives nan for all initial guess... FAILED tests.py::test_tracks - tarfile.ReadError: file could not be opened su... ==================== 4 failed, 3 passed, 1 warning in 6.18s ====================

timothydmorton / isochrones

pytest: eep0 issues #142