arrow-py / arrow

🏹 Better dates & times for Python
https://arrow.readthedocs.io
Apache License 2.0
8.67k stars 668 forks source link

1.1.1: pytest is failing when `pytest-randomly` is used #1038

Open kloczek opened 3 years ago

kloczek commented 3 years ago

System Info

Looks like pytest is failing when pytest-randomly is installed. Example:

+ PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-arrow-1.1.1-2.1.fc35.x86_64/usr/lib64/python3.8/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-arrow-1.1.1-2.1.fc35.x86_64/usr/lib/python3.8/site-packages
+ /usr/bin/pytest -ra -q
=========================================================================== test session starts ============================================================================
platform linux -- Python 3.8.12, pytest-6.2.5, py-1.10.0, pluggy-0.13.1
benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
Using --randomly-seed=457307952
rootdir: /home/tkloczko/rpmbuild/BUILD/arrow-1.1.1, configfile: tox.ini, testpaths: tests
plugins: forked-1.3.0, shutil-1.7.0, virtualenv-1.7.0, expect-1.1.0, flake8-1.0.7, timeout-1.4.2, betamax-0.8.1, freezegun-0.4.2, aspectlib-1.5.2, toolbox-0.5, rerunfailures-9.1.1, requests-mock-1.9.3, cov-2.12.1, flaky-3.7.0, benchmark-3.4.1, xdist-2.3.0, pylama-7.7.1, datadir-1.3.1, regressions-2.2.0, cases-3.6.3, xprocess-0.18.1, black-0.3.12, anyio-3.3.0, asyncio-0.15.1, trio-0.7.0, subtests-0.5.0, isort-2.0.0, hypothesis-6.14.6, mock-3.6.1, profiling-1.7.0, randomly-3.8.0, Faker-8.12.1, nose2pytest-1.0.8, pyfakefs-4.5.1, tornado-0.8.1, twisted-1.13.3
collected 1737 items

tests/test_factory.py ................................................                                                                                               [  2%]
tests/test_util.py .....                                                                                                                                             [  3%]
tests/test_formatter.py ............................................................................................................................................ [ 11%]
.................................................................................................................................................................... [ 20%]
.................................................................................................................................................................... [ 29%]
.......................................................................................................................................................              [ 38%]
tests/test_arrow.py ............X..............x.................................................................................................................... [ 46%]
....................................................................                                                                                                 [ 50%]
tests/test_api.py ....                                                                                                                                               [ 51%]
tests/test_parser.py ............................................................................................................................................... [ 59%]
.................................................................................................................................................................... [ 68%]
.................................................................................................................................................................... [ 78%]
.................................................................................................................................................................... [ 87%]
..................................................................................                                                                                   [ 92%]
tests/test_locales.py ...................................................................................................FF...............................           [100%]

================================================================================= FAILURES =================================================================================
_____________________________________________________________ TestLocaleValidation.test_locale_name_validation _____________________________________________________________

self = <tests.test_locales.TestLocaleValidation object at 0x7f0d3f734310>

    def test_locale_name_validation(self):

        for locale_cls in self.locales.values():
            for locale_name in locale_cls.names:
>               assert len(locale_name) == 2 or len(locale_name) == 5
E               AssertionError: assert (3 == 2 or 3 == 5)
E                +  where 3 = len('foo')
E                +  and   3 = len('foo')

tests/test_locales.py:40: AssertionError
_______________________________________________________________ TestLocaleValidation.test_locale_validation ________________________________________________________________

self = <tests.test_locales.TestLocaleValidation object at 0x7f0d3fae9a00>

    def test_locale_validation(self):

        for locale_cls in self.locales.values():
            # 7 days + 1 spacer to allow for 1-indexing of months
>           assert len(locale_cls.day_names) == 8
E           AssertionError: assert 0 == 8
E            +  where 0 = len([])
E            +    where [] = <class 'tests.test_locales.TestCustomLocale.test_custom_locale_subclass.<locals>.CustomLocale1'>.day_names

tests/test_locales.py:14: AssertionError
============================================================================= warnings summary =============================================================================
../../../../../usr/lib/python3.8/site-packages/dateutil/zoneinfo/__init__.py:26
  /usr/lib/python3.8/site-packages/dateutil/zoneinfo/__init__.py:26: UserWarning: I/O error(2): No such file or directory
    warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))

-- Docs: https://docs.pytest.org/en/stable/warnings.html

---------- coverage: platform linux, python 3.8.12-final-0 -----------
Name                 Stmts   Miss Branch BrPart  Cover   Missing
----------------------------------------------------------------
arrow/__init__.py        7      0      0      0   100%
arrow/_version.py        1      0      0      0   100%
arrow/api.py            30      0      0      0   100%
arrow/arrow.py         550      0    239      0   100%
arrow/constants.py      11      0      0      0   100%
arrow/factory.py        86      0     48      0   100%
arrow/formatter.py     102      0     72      0   100%
arrow/locales.py       965      0    144      0   100%
arrow/parser.py        313      0    142      0   100%
arrow/util.py           45      0     22      0   100%
----------------------------------------------------------------
TOTAL                 2110      0    667      0   100%
Coverage XML written to file coverage.xml

Required test coverage of 99% reached. Total coverage: 100.00%
========================================================================= short test summary info ==========================================================================
XFAIL tests/test_arrow.py::TestArrowHumanize::test_month
  known issue with humanize month limits
XPASS tests/test_arrow.py::TestArrowHumanize::test_months known issue with humanize month limits
FAILED tests/test_locales.py::TestLocaleValidation::test_locale_name_validation - AssertionError: assert (3 == 2 or 3 == 5)
FAILED tests/test_locales.py::TestLocaleValidation::test_locale_validation - AssertionError: assert 0 == 8
===================================================== 2 failed, 1733 passed, 1 xfailed, 1 xpassed, 1 warning in 14.78s =====================================================
pytest-xprocess reminder::Be sure to terminate the started process by running 'pytest --xkill' if you have not explicitly done so in your fixture with 'xprocess.getinfo(<process_name>).terminate()'.

As long as randomly plugin is disaled everything is fine:

+ PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-arrow-1.1.1-2.1.fc35.x86_64/usr/lib64/python3.8/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-arrow-1.1.1-2.1.fc35.x86_64/usr/lib/python3.8/site-packages
+ /usr/bin/pytest -ra -q -p no:randomly
=========================================================================== test session starts ============================================================================
platform linux -- Python 3.8.12, pytest-6.2.5, py-1.10.0, pluggy-0.13.1
benchmark: 3.4.1 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000)
rootdir: /home/tkloczko/rpmbuild/BUILD/arrow-1.1.1, configfile: tox.ini, testpaths: tests
plugins: forked-1.3.0, shutil-1.7.0, virtualenv-1.7.0, expect-1.1.0, flake8-1.0.7, timeout-1.4.2, betamax-0.8.1, freezegun-0.4.2, aspectlib-1.5.2, toolbox-0.5, rerunfailures-9.1.1, requests-mock-1.9.3, cov-2.12.1, flaky-3.7.0, benchmark-3.4.1, xdist-2.3.0, pylama-7.7.1, datadir-1.3.1, regressions-2.2.0, cases-3.6.3, xprocess-0.18.1, black-0.3.12, anyio-3.3.0, asyncio-0.15.1, trio-0.7.0, subtests-0.5.0, isort-2.0.0, hypothesis-6.14.6, mock-3.6.1, profiling-1.7.0, Faker-8.12.1, nose2pytest-1.0.8, pyfakefs-4.5.1, tornado-0.8.1, twisted-1.13.3
collected 1737 items

tests/test_api.py ....                                                                                                                                               [  0%]
tests/test_arrow.py ................................................................................................................................................ [  8%]
....................x.X.............................................                                                                                                 [ 12%]
tests/test_factory.py ................................................                                                                                               [ 15%]
tests/test_formatter.py ............................................................................................................................................ [ 23%]
.................................................................................................................................................................... [ 32%]
.................................................................................................................................................................... [ 42%]
.......................................................................................................................................................              [ 50%]
tests/test_locales.py ....................................................................................................................................           [ 58%]
tests/test_parser.py ............................................................................................................................................... [ 66%]
.................................................................................................................................................................... [ 76%]
.................................................................................................................................................................... [ 85%]
.................................................................................................................................................................... [ 94%]
..................................................................................                                                                                   [ 99%]
tests/test_util.py .....                                                                                                                                             [100%]

============================================================================= warnings summary =============================================================================
../../../../../usr/lib/python3.8/site-packages/dateutil/zoneinfo/__init__.py:26
  /usr/lib/python3.8/site-packages/dateutil/zoneinfo/__init__.py:26: UserWarning: I/O error(2): No such file or directory
    warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))

-- Docs: https://docs.pytest.org/en/stable/warnings.html

---------- coverage: platform linux, python 3.8.12-final-0 -----------
Name                 Stmts   Miss Branch BrPart  Cover   Missing
----------------------------------------------------------------
arrow/__init__.py        7      0      0      0   100%
arrow/_version.py        1      0      0      0   100%
arrow/api.py            30      0      0      0   100%
arrow/arrow.py         550      0    239      0   100%
arrow/constants.py      11      0      0      0   100%
arrow/factory.py        86      0     48      0   100%
arrow/formatter.py     102      0     72      0   100%
arrow/locales.py       965      0    144      0   100%
arrow/parser.py        313      0    142      0   100%
arrow/util.py           45      0     22      0   100%
----------------------------------------------------------------
TOTAL                 2110      0    667      0   100%
Coverage XML written to file coverage.xml

Required test coverage of 99% reached. Total coverage: 100.00%
========================================================================= short test summary info ==========================================================================
XFAIL tests/test_arrow.py::TestArrowHumanize::test_month
  known issue with humanize month limits
XPASS tests/test_arrow.py::TestArrowHumanize::test_months known issue with humanize month limits
========================================================== 1735 passed, 1 xfailed, 1 xpassed, 1 warning in 12.92s ==========================================================
pytest-xprocess reminder::Be sure to terminate the started process by running 'pytest --xkill' if you have not explicitly done so in your fixture with 'xprocess.getinfo(<process_name>).terminate()'.

To diagnose that kind of issues https://github.com/mrbean-bremen/pytest-find-dependencies/ can be used.

systemcatch commented 3 years ago

Thanks for this @kloczek, after running pytest --find-dependencies I get the following result.

=====================================================
Run dependency analysis for 1750 tests.
Executed 5247 tests in 24 test runs.
Dependent tests:
  tests/test_locales.py::TestLocaleValidation::test_locale_name_validation depends on tests/test_locales.py::TestCustomLocale::test_custom_locale_subclass
  tests/test_locales.py::TestLocaleValidation::test_locale_validation depends on tests/test_locales.py::TestCustomLocale::test_custom_locale_subclass
=====================================================

Stated dependency.

class TestCustomLocale:
    def test_custom_locale_subclass(self):
        class CustomLocale1(locales.Locale):
            names = ["foo", "foo-BAR"]

        assert locales.get_locale("foo") is not None
        assert locales.get_locale("foo-BAR") is not None
        assert locales.get_locale("foo_bar") is not None

        class CustomLocale2(locales.Locale):
            names = ["underscores_ok"]

        assert locales.get_locale("underscores_ok") is not None

Dependent tests

    def test_locale_validation(self):

        for locale_cls in self.locales.values():
            # 7 days + 1 spacer to allow for 1-indexing of months
            assert len(locale_cls.day_names) == 8
            assert locale_cls.day_names[0] == ""
            # ensure that all string from index 1 onward are valid (not blank or None)
            assert all(locale_cls.day_names[1:])

            assert len(locale_cls.day_abbreviations) == 8
            assert locale_cls.day_abbreviations[0] == ""
            assert all(locale_cls.day_abbreviations[1:])

            # 12 months + 1 spacer to allow for 1-indexing of months
            assert len(locale_cls.month_names) == 13
            assert locale_cls.month_names[0] == ""
            assert all(locale_cls.month_names[1:])

            assert len(locale_cls.month_abbreviations) == 13
            assert locale_cls.month_abbreviations[0] == ""
            assert all(locale_cls.month_abbreviations[1:])

            assert len(locale_cls.names) > 0
            assert locale_cls.past is not None
            assert locale_cls.future is not None

    def test_locale_name_validation(self):

        for locale_cls in self.locales.values():
            for locale_name in locale_cls.names:
                assert len(locale_name) == 2 or len(locale_name) == 5
                assert locale_name.islower()
                # Not a two-letter code
                if len(locale_name) > 2:
                    assert "-" in locale_name
                    assert locale_name.count("-") == 1