pytest-dev / pytest-rerunfailures

a pytest plugin that re-runs failed tests up to -n times to eliminate flakey failures
Other
380 stars 83 forks source link

14.0: pytest 8.2.2 fails in 5 units #269

Closed kloczek closed 3 months ago

kloczek commented 3 months ago

I'm packaging your module as an rpm package so I'm using the typical PEP517 based build, install and test cycle used on building packages from non-root account.

Here is pytest output: ```console + PYTHONPATH=/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-rerunfailures-14.0-2.fc37.x86_64/usr/lib64/python3.10/site-packages:/home/tkloczko/rpmbuild/BUILDROOT/python-pytest-rerunfailures-14.0-2.fc37.x86_64/usr/lib/python3.10/site-packages + /usr/bin/pytest -ra -m 'not network' -p no:randomly ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0 configfile: pyproject.toml plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 109 items tests/test_pytest_rerunfailures.py ..................................... [ 33%] .................................................................FFFF..F [100%] =================================== FAILURES =================================== _________________ test_run_session_teardown_once_after_reruns __________________ testdir = def test_run_session_teardown_once_after_reruns(testdir): testdir.makepyfile( """ import logging import pytest from unittest import TestCase @pytest.fixture(scope='session', autouse=True) def session_fixture(): logging.info('session setup') yield logging.info('session teardown') @pytest.fixture(scope='class', autouse=True) def class_fixture(): logging.info('class setup') yield logging.info('class teardown') @pytest.fixture(scope='function', autouse=True) def function_fixture(): logging.info('function setup') yield logging.info('function teardown') @pytest.fixture(scope='function') def function_skip_fixture(): logging.info('skip fixture setup') pytest.skip('some reason') yield logging.info('skip fixture teardown') @pytest.fixture(scope='function') def function_setup_fail_fixture(): logging.info('fail fixture setup') assert False yield logging.info('fail fixture teardown') class TestFirstPassLastFail: @staticmethod def test_1(): logging.info("TestFirstPassLastFail 1") @staticmethod def test_2(): logging.info("TestFirstPassLastFail 2") assert False class TestFirstFailLastPass: @staticmethod def test_1(): logging.info("TestFirstFailLastPass 1") assert False @staticmethod def test_2(): logging.info("TestFirstFailLastPass 2") class TestSkipFirst: @staticmethod @pytest.mark.skipif(True, reason='Some reason') def test_1(): logging.info("TestSkipFirst 1") assert False @staticmethod def test_2(): logging.info("TestSkipFirst 2") assert False class TestSkipLast: @staticmethod def test_1(): logging.info("TestSkipLast 1") assert False @staticmethod @pytest.mark.skipif(True, reason='Some reason') def test_2(): logging.info("TestSkipLast 2") assert False class TestSkipFixture: @staticmethod def test_1(function_skip_fixture): logging.info("TestSkipFixture 1") class TestSetupFailed: @staticmethod def test_1(function_setup_fail_fixture): logging.info("TestSetupFailed 1") class TestTestCaseFailFirstFailLast(TestCase): @staticmethod def test_1(): logging.info("TestTestCaseFailFirstFailLast 1") assert False @staticmethod def test_2(): logging.info("TestTestCaseFailFirstFailLast 2") assert False class TestTestCaseSkipFirst(TestCase): @staticmethod @pytest.mark.skipif(True, reason='Some reason') def test_1(): logging.info("TestTestCaseSkipFirst 1") assert False @staticmethod def test_2(): logging.info("TestTestCaseSkipFirst 2") assert False class TestTestCaseSkipLast(TestCase): @staticmethod def test_1(): logging.info("TestTestCaseSkipLast 1") assert False @staticmethod @pytest.mark.skipif(True, reason="Some reason") def test_2(): logging.info("TestTestCaseSkipLast 2") assert False""" ) import logging logging.info = mock.MagicMock() result = testdir.runpytest("--reruns", "2") expected_calls = [ mock.call("session setup"), # TestFirstPassLastFail mock.call("class setup"), mock.call("function setup"), mock.call("TestFirstPassLastFail 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstPassLastFail 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstPassLastFail 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstPassLastFail 2"), mock.call("function teardown"), mock.call("class teardown"), # TestFirstFailLastPass mock.call("class setup"), mock.call("function setup"), mock.call("TestFirstFailLastPass 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstFailLastPass 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstFailLastPass 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestFirstFailLastPass 2"), mock.call("function teardown"), mock.call("class teardown"), # TestSkipFirst mock.call("class setup"), mock.call("function setup"), mock.call("TestSkipFirst 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestSkipFirst 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestSkipFirst 2"), mock.call("function teardown"), mock.call("class teardown"), # TestSkipLast mock.call("class setup"), mock.call("function setup"), mock.call("TestSkipLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestSkipLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestSkipLast 1"), mock.call("function teardown"), mock.call("class teardown"), # TestSkipFixture mock.call("class setup"), mock.call("function setup"), mock.call("skip fixture setup"), mock.call("function teardown"), mock.call("class teardown"), # TestSetupFailed mock.call("class setup"), mock.call("function setup"), mock.call("fail fixture setup"), mock.call("function teardown"), mock.call("function setup"), mock.call("fail fixture setup"), mock.call("function teardown"), mock.call("function setup"), mock.call("fail fixture setup"), mock.call("function teardown"), mock.call("class teardown"), # TestTestCaseFailFirstFailLast mock.call("class setup"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseFailFirstFailLast 2"), mock.call("function teardown"), mock.call("class teardown"), # TestTestCaseSkipFirst mock.call("class setup"), mock.call("function setup"), mock.call("TestTestCaseSkipFirst 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseSkipFirst 2"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseSkipFirst 2"), mock.call("function teardown"), mock.call("class teardown"), # TestTestCaseSkipLast mock.call("class setup"), mock.call("function setup"), mock.call("TestTestCaseSkipLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseSkipLast 1"), mock.call("function teardown"), mock.call("function setup"), mock.call("TestTestCaseSkipLast 1"), mock.call("function teardown"), mock.call("class teardown"), mock.call("session teardown"), ] > logging.info.assert_has_calls(expected_calls, any_order=False) /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0/tests/test_pytest_rerunfailures.py:1084: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = calls = [call('session setup'), call('class setup'), call('function setup'), call('TestFirstPassLastFail 1'), call('function teardown'), call('function setup'), ...] any_order = False def assert_has_calls(self, calls, any_order=False): """assert the mock has been called with the specified calls. The `mock_calls` list is checked for the calls. If `any_order` is False (the default) then the calls must be sequential. There can be extra calls before or after the specified calls. If `any_order` is True then the calls can be in any order, but they must all appear in `mock_calls`.""" expected = [self._call_matcher(c) for c in calls] cause = next((e for e in expected if isinstance(e, Exception)), None) all_calls = _CallList(self._call_matcher(c) for c in self.mock_calls) if not any_order: if expected not in all_calls: if cause is None: problem = 'Calls not found.' else: problem = ('Error processing expected calls.\n' 'Errors: {}').format( [e if isinstance(e, Exception) else None for e in expected]) > raise AssertionError( f'{problem}\n' f'Expected: {_CallList(calls)}' f'{self._calls_repr(prefix="Actual").rstrip(".")}' ) from cause E AssertionError: Calls not found. E Expected: [call('session setup'), E call('class setup'), E call('function setup'), E call('TestFirstPassLastFail 1'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 2'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('skip fixture setup'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 1'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 1'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 1'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 2'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 2'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 2'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestTestCaseSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipFirst 2'), E call('function teardown'), E call('class teardown'), E call('class setup'), E call('function setup'), E call('TestTestCaseSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipLast 1'), E call('function teardown'), E call('class teardown'), E call('session teardown')] E Actual: [call('session setup'), E call('class setup'), E call('function setup'), E call('TestFirstPassLastFail 1'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('function setup'), E call('TestFirstPassLastFail 2'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 1'), E call('function teardown'), E call('function setup'), E call('TestFirstFailLastPass 2'), E call('function teardown'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('function setup'), E call('TestSkipLast 1'), E call('function teardown'), E call('function setup'), E call('skip fixture setup'), E call('function teardown'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('function setup'), E call('fail fixture setup'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 1'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('TestTestCaseFailFirstFailLast 2'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipFirst 2'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('TestTestCaseSkipLast 1'), E call('function teardown'), E call('function setup'), E call('function teardown'), E call('function setup'), E call('function teardown')] /usr/lib64/python3.10/unittest/mock.py:966: AssertionError ----------------------------- Captured stdout call ----------------------------- ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_run_session_teardown_once_after_reruns0 plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 16 items test_run_session_teardown_once_after_reruns.py .RRFRRF.sRRFRRFssRRERRFRR [ 75%] FsRRFRRFs [100%] ==================================== ERRORS ==================================== ___________________ ERROR at setup of TestSetupFailed.test_1 ___________________ @pytest.fixture(scope='function') def function_setup_fail_fixture(): logging.info('fail fixture setup') > assert False E assert False test_run_session_teardown_once_after_reruns.py:34: AssertionError =================================== FAILURES =================================== _________________________ TestFirstPassLastFail.test_2 _________________________ @staticmethod def test_2(): logging.info("TestFirstPassLastFail 2") > assert False E assert False test_run_session_teardown_once_after_reruns.py:47: AssertionError _________________________ TestFirstFailLastPass.test_1 _________________________ @staticmethod def test_1(): logging.info("TestFirstFailLastPass 1") > assert False E assert False test_run_session_teardown_once_after_reruns.py:54: AssertionError _____________________________ TestSkipFirst.test_2 _____________________________ @staticmethod def test_2(): logging.info("TestSkipFirst 2") > assert False E assert False test_run_session_teardown_once_after_reruns.py:70: AssertionError _____________________________ TestSkipLast.test_1 ______________________________ @staticmethod def test_1(): logging.info("TestSkipLast 1") > assert False E assert False test_run_session_teardown_once_after_reruns.py:76: AssertionError _____________________ TestTestCaseFailFirstFailLast.test_1 _____________________ cls = func = . at 0x7f2c75fb3e20> when = 'call' reraise = (, ) @classmethod def from_call( cls, func: Callable[[], TResult], when: Literal["collect", "setup", "call", "teardown"], reraise: Optional[ Union[Type[BaseException], Tuple[Type[BaseException], ...]] ] = None, ) -> "CallInfo[TResult]": """Call func, wrapping the result in a CallInfo. :param func: The function to call. Called without arguments. :param when: The phase in which the function is called. :param reraise: Exception or exceptions that shall propagate if raised by the function, instead of being wrapped in the CallInfo. """ excinfo = None start = timing.time() precise_start = timing.perf_counter() try: > result: Optional[TResult] = func() /usr/lib/python3.10/site-packages/_pytest/runner.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.10/site-packages/_pytest/runner.py:241: in lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise /usr/lib/python3.10/site-packages/pluggy/_hooks.py:513: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:480: in traced_hookexec return outcome.get_result() /usr/lib/python3.10/site-packages/pluggy/_manager.py:477: in lambda: oldcall(hook_name, hook_impls, caller_kwargs, firstresult) /usr/lib/python3.10/site-packages/_pytest/threadexception.py:87: in pytest_runtest_call yield from thread_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/threadexception.py:63: in thread_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:90: in pytest_runtest_call yield from unraisable_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:65: in unraisable_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/logging.py:850: in pytest_runtest_call yield from self._runtest_for(item, "call") /usr/lib/python3.10/site-packages/_pytest/logging.py:833: in _runtest_for yield /usr/lib/python3.10/site-packages/_pytest/capture.py:878: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/skipping.py:257: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/runner.py:183: in pytest_runtest_call raise e /usr/lib/python3.10/site-packages/_pytest/runner.py:173: in pytest_runtest_call item.runtest() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def runtest(self) -> None: from _pytest.debugging import maybe_wrap_pytest_function_for_tracing testcase = self.instance > assert testcase is not None E AssertionError /usr/lib/python3.10/site-packages/_pytest/unittest.py:319: AssertionError _____________________ TestTestCaseFailFirstFailLast.test_2 _____________________ cls = func = . at 0x7f2c77f3c160> when = 'call' reraise = (, ) @classmethod def from_call( cls, func: Callable[[], TResult], when: Literal["collect", "setup", "call", "teardown"], reraise: Optional[ Union[Type[BaseException], Tuple[Type[BaseException], ...]] ] = None, ) -> "CallInfo[TResult]": """Call func, wrapping the result in a CallInfo. :param func: The function to call. Called without arguments. :param when: The phase in which the function is called. :param reraise: Exception or exceptions that shall propagate if raised by the function, instead of being wrapped in the CallInfo. """ excinfo = None start = timing.time() precise_start = timing.perf_counter() try: > result: Optional[TResult] = func() /usr/lib/python3.10/site-packages/_pytest/runner.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.10/site-packages/_pytest/runner.py:241: in lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise /usr/lib/python3.10/site-packages/pluggy/_hooks.py:513: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:480: in traced_hookexec return outcome.get_result() /usr/lib/python3.10/site-packages/pluggy/_manager.py:477: in lambda: oldcall(hook_name, hook_impls, caller_kwargs, firstresult) /usr/lib/python3.10/site-packages/_pytest/threadexception.py:87: in pytest_runtest_call yield from thread_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/threadexception.py:63: in thread_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:90: in pytest_runtest_call yield from unraisable_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:65: in unraisable_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/logging.py:850: in pytest_runtest_call yield from self._runtest_for(item, "call") /usr/lib/python3.10/site-packages/_pytest/logging.py:833: in _runtest_for yield /usr/lib/python3.10/site-packages/_pytest/capture.py:878: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/skipping.py:257: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/runner.py:183: in pytest_runtest_call raise e /usr/lib/python3.10/site-packages/_pytest/runner.py:173: in pytest_runtest_call item.runtest() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def runtest(self) -> None: from _pytest.debugging import maybe_wrap_pytest_function_for_tracing testcase = self.instance > assert testcase is not None E AssertionError /usr/lib/python3.10/site-packages/_pytest/unittest.py:319: AssertionError _________________________ TestTestCaseSkipFirst.test_2 _________________________ cls = func = . at 0x7f2c77de3c70> when = 'call' reraise = (, ) @classmethod def from_call( cls, func: Callable[[], TResult], when: Literal["collect", "setup", "call", "teardown"], reraise: Optional[ Union[Type[BaseException], Tuple[Type[BaseException], ...]] ] = None, ) -> "CallInfo[TResult]": """Call func, wrapping the result in a CallInfo. :param func: The function to call. Called without arguments. :param when: The phase in which the function is called. :param reraise: Exception or exceptions that shall propagate if raised by the function, instead of being wrapped in the CallInfo. """ excinfo = None start = timing.time() precise_start = timing.perf_counter() try: > result: Optional[TResult] = func() /usr/lib/python3.10/site-packages/_pytest/runner.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.10/site-packages/_pytest/runner.py:241: in lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise /usr/lib/python3.10/site-packages/pluggy/_hooks.py:513: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:480: in traced_hookexec return outcome.get_result() /usr/lib/python3.10/site-packages/pluggy/_manager.py:477: in lambda: oldcall(hook_name, hook_impls, caller_kwargs, firstresult) /usr/lib/python3.10/site-packages/_pytest/threadexception.py:87: in pytest_runtest_call yield from thread_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/threadexception.py:63: in thread_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:90: in pytest_runtest_call yield from unraisable_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:65: in unraisable_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/logging.py:850: in pytest_runtest_call yield from self._runtest_for(item, "call") /usr/lib/python3.10/site-packages/_pytest/logging.py:833: in _runtest_for yield /usr/lib/python3.10/site-packages/_pytest/capture.py:878: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/skipping.py:257: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/runner.py:183: in pytest_runtest_call raise e /usr/lib/python3.10/site-packages/_pytest/runner.py:173: in pytest_runtest_call item.runtest() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def runtest(self) -> None: from _pytest.debugging import maybe_wrap_pytest_function_for_tracing testcase = self.instance > assert testcase is not None E AssertionError /usr/lib/python3.10/site-packages/_pytest/unittest.py:319: AssertionError _________________________ TestTestCaseSkipLast.test_1 __________________________ cls = func = . at 0x7f2c77de29e0> when = 'call' reraise = (, ) @classmethod def from_call( cls, func: Callable[[], TResult], when: Literal["collect", "setup", "call", "teardown"], reraise: Optional[ Union[Type[BaseException], Tuple[Type[BaseException], ...]] ] = None, ) -> "CallInfo[TResult]": """Call func, wrapping the result in a CallInfo. :param func: The function to call. Called without arguments. :param when: The phase in which the function is called. :param reraise: Exception or exceptions that shall propagate if raised by the function, instead of being wrapped in the CallInfo. """ excinfo = None start = timing.time() precise_start = timing.perf_counter() try: > result: Optional[TResult] = func() /usr/lib/python3.10/site-packages/_pytest/runner.py:341: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /usr/lib/python3.10/site-packages/_pytest/runner.py:241: in lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise /usr/lib/python3.10/site-packages/pluggy/_hooks.py:513: in __call__ return self._hookexec(self.name, self._hookimpls.copy(), kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:120: in _hookexec return self._inner_hookexec(hook_name, methods, kwargs, firstresult) /usr/lib/python3.10/site-packages/pluggy/_manager.py:480: in traced_hookexec return outcome.get_result() /usr/lib/python3.10/site-packages/pluggy/_manager.py:477: in lambda: oldcall(hook_name, hook_impls, caller_kwargs, firstresult) /usr/lib/python3.10/site-packages/_pytest/threadexception.py:87: in pytest_runtest_call yield from thread_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/threadexception.py:63: in thread_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:90: in pytest_runtest_call yield from unraisable_exception_runtest_hook() /usr/lib/python3.10/site-packages/_pytest/unraisableexception.py:65: in unraisable_exception_runtest_hook yield /usr/lib/python3.10/site-packages/_pytest/logging.py:850: in pytest_runtest_call yield from self._runtest_for(item, "call") /usr/lib/python3.10/site-packages/_pytest/logging.py:833: in _runtest_for yield /usr/lib/python3.10/site-packages/_pytest/capture.py:878: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/skipping.py:257: in pytest_runtest_call return (yield) /usr/lib/python3.10/site-packages/_pytest/runner.py:183: in pytest_runtest_call raise e /usr/lib/python3.10/site-packages/_pytest/runner.py:173: in pytest_runtest_call item.runtest() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = def runtest(self) -> None: from _pytest.debugging import maybe_wrap_pytest_function_for_tracing testcase = self.instance > assert testcase is not None E AssertionError /usr/lib/python3.10/site-packages/_pytest/unittest.py:319: AssertionError =========================== short test summary info ============================ FAILED test_run_session_teardown_once_after_reruns.py::TestFirstPassLastFail::test_2 FAILED test_run_session_teardown_once_after_reruns.py::TestFirstFailLastPass::test_1 FAILED test_run_session_teardown_once_after_reruns.py::TestSkipFirst::test_2 FAILED test_run_session_teardown_once_after_reruns.py::TestSkipLast::test_1 FAILED test_run_session_teardown_once_after_reruns.py::TestTestCaseFailFirstFailLast::test_1 FAILED test_run_session_teardown_once_after_reruns.py::TestTestCaseFailFirstFailLast::test_2 FAILED test_run_session_teardown_once_after_reruns.py::TestTestCaseSkipFirst::test_2 FAILED test_run_session_teardown_once_after_reruns.py::TestTestCaseSkipLast::test_1 ERROR test_run_session_teardown_once_after_reruns.py::TestSetupFailed::test_1 ========== 8 failed, 2 passed, 5 skipped, 1 error, 18 rerun in 1.04s =========== __________________ test_exception_matches_rerun_except_query ___________________ testdir = def test_exception_matches_rerun_except_query(testdir): testdir.makepyfile( """ import pytest @pytest.fixture(scope="session", autouse=True) def session_fixture(): print("session setup") yield "session" print("session teardown") @pytest.fixture(scope="package", autouse=True) def package_fixture(): print("package setup") yield "package" print("package teardown") @pytest.fixture(scope="module", autouse=True) def module_fixture(): print("module setup") yield "module" print("module teardown") @pytest.fixture(scope="class", autouse=True) def class_fixture(): print("class setup") yield "class" print("class teardown") @pytest.fixture(scope="function", autouse=True) def function_fixture(): print("function setup") yield "function" print("function teardown") @pytest.mark.flaky(reruns=1, rerun_except=["AssertionError"]) class TestStuff: def test_1(self): raise AssertionError("fail") def test_2(self): assert False """ ) result = testdir.runpytest() assert_outcomes(result, passed=0, failed=2, rerun=1) > result.stdout.fnmatch_lines("session teardown") E Failed: nomatch: 'session teardown' E and: '============================= test session starts ==============================' E and: 'platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0' E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_matches_rerun_except_query0' E and: 'plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1' E and: 'collected 2 items' E and: '' E and: 'test_exception_matches_rerun_except_query.py FR [100%]F [100%]' E and: '' E and: '=================================== FAILURES ===================================' E and: '_______________________________ TestStuff.test_1 _______________________________' E and: '' E and: 'self = ' E and: '' E and: ' def test_1(self):' E and: '> raise AssertionError("fail")' E and: 'E AssertionError: fail' E and: '' E and: 'test_exception_matches_rerun_except_query.py:36: AssertionError' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'session setup' E and: 'package setup' E and: 'module setup' E and: 'class setup' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '_______________________________ TestStuff.test_2 _______________________________' E and: '' E and: 'self = ' E and: '' E and: ' def test_2(self):' E and: '> assert False' E and: 'E assert False' E and: '' E and: 'test_exception_matches_rerun_except_query.py:39: AssertionError' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '=========================== short test summary info ============================' E and: 'FAILED test_exception_matches_rerun_except_query.py::TestStuff::test_1 - Asse...' E and: 'FAILED test_exception_matches_rerun_except_query.py::TestStuff::test_2 - asse...' E and: '========================== 2 failed, 1 rerun in 0.02s ==========================' E remains unmatched: 'session teardown' /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0/tests/test_pytest_rerunfailures.py:1135: Failed ----------------------------- Captured stdout call ----------------------------- ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_matches_rerun_except_query0 plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 2 items test_exception_matches_rerun_except_query.py FR [100%]F [100%] =================================== FAILURES =================================== _______________________________ TestStuff.test_1 _______________________________ self = def test_1(self): > raise AssertionError("fail") E AssertionError: fail test_exception_matches_rerun_except_query.py:36: AssertionError ---------------------------- Captured stdout setup ----------------------------- session setup package setup module setup class setup function setup --------------------------- Captured stdout teardown --------------------------- function teardown _______________________________ TestStuff.test_2 _______________________________ self = def test_2(self): > assert False E assert False test_exception_matches_rerun_except_query.py:39: AssertionError ---------------------------- Captured stdout setup ----------------------------- function setup --------------------------- Captured stdout teardown --------------------------- function teardown ---------------------------- Captured stdout setup ----------------------------- function setup --------------------------- Captured stdout teardown --------------------------- function teardown --------------------------- Captured stdout teardown --------------------------- function teardown =========================== short test summary info ============================ FAILED test_exception_matches_rerun_except_query.py::TestStuff::test_1 - Asse... FAILED test_exception_matches_rerun_except_query.py::TestStuff::test_2 - asse... ========================== 2 failed, 1 rerun in 0.02s ========================== _________________ test_exception_not_match_rerun_except_query __________________ testdir = def test_exception_not_match_rerun_except_query(testdir): testdir.makepyfile( """ import pytest @pytest.fixture(scope="session", autouse=True) def session_fixture(): print("session setup") yield "session" print("session teardown") @pytest.fixture(scope="function", autouse=True) def function_fixture(): print("function setup") yield "function" print("function teardown") @pytest.mark.flaky(reruns=1, rerun_except="AssertionError") def test_1(session_fixture, function_fixture): raise ValueError("value") """ ) result = testdir.runpytest() assert_outcomes(result, passed=0, failed=1, rerun=1) > result.stdout.fnmatch_lines("session teardown") E Failed: nomatch: 'session teardown' E and: '============================= test session starts ==============================' E and: 'platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0' E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_not_match_rerun_except_query0' E and: 'plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1' E and: 'collected 1 item' E and: '' E and: 'test_exception_not_match_rerun_except_query.py R [100%]F [100%]' E and: '' E and: '=================================== FAILURES ===================================' E and: '____________________________________ test_1 ____________________________________' E and: '' E and: "session_fixture = 'session', function_fixture = 'function'" E and: '' E and: ' @pytest.mark.flaky(reruns=1, rerun_except="AssertionError")' E and: ' def test_1(session_fixture, function_fixture):' E and: '> raise ValueError("value")' E and: 'E ValueError: value' E and: '' E and: 'test_exception_not_match_rerun_except_query.py:17: ValueError' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'session setup' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '=========================== short test summary info ============================' E and: 'FAILED test_exception_not_match_rerun_except_query.py::test_1 - ValueError: v...' E and: '========================== 1 failed, 1 rerun in 0.04s ==========================' E remains unmatched: 'session teardown' /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0/tests/test_pytest_rerunfailures.py:1166: Failed ----------------------------- Captured stdout call ----------------------------- ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_not_match_rerun_except_query0 plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 1 item test_exception_not_match_rerun_except_query.py R [100%]F [100%] =================================== FAILURES =================================== ____________________________________ test_1 ____________________________________ session_fixture = 'session', function_fixture = 'function' @pytest.mark.flaky(reruns=1, rerun_except="AssertionError") def test_1(session_fixture, function_fixture): > raise ValueError("value") E ValueError: value test_exception_not_match_rerun_except_query.py:17: ValueError ---------------------------- Captured stdout setup ----------------------------- session setup function setup --------------------------- Captured stdout teardown --------------------------- function teardown ---------------------------- Captured stdout setup ----------------------------- function setup --------------------------- Captured stdout teardown --------------------------- function teardown --------------------------- Captured stdout teardown --------------------------- function teardown =========================== short test summary info ============================ FAILED test_exception_not_match_rerun_except_query.py::test_1 - ValueError: v... ========================== 1 failed, 1 rerun in 0.04s ========================== ___________________ test_exception_matches_only_rerun_query ____________________ testdir = def test_exception_matches_only_rerun_query(testdir): testdir.makepyfile( """ import pytest @pytest.fixture(scope="session", autouse=True) def session_fixture(): print("session setup") yield "session" print("session teardown") @pytest.fixture(scope="function", autouse=True) def function_fixture(): print("function setup") yield "function" print("function teardown") @pytest.mark.flaky(reruns=1, only_rerun=["AssertionError"]) def test_1(session_fixture, function_fixture): raise AssertionError("fail") """ ) result = testdir.runpytest() assert_outcomes(result, passed=0, failed=1, rerun=1) > result.stdout.fnmatch_lines("session teardown") E Failed: nomatch: 'session teardown' E and: '============================= test session starts ==============================' E and: 'platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0' E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_matches_only_rerun_query0' E and: 'plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1' E and: 'collected 1 item' E and: '' E and: 'test_exception_matches_only_rerun_query.py R [100%]F [100%]' E and: '' E and: '=================================== FAILURES ===================================' E and: '____________________________________ test_1 ____________________________________' E and: '' E and: "session_fixture = 'session', function_fixture = 'function'" E and: '' E and: ' @pytest.mark.flaky(reruns=1, only_rerun=["AssertionError"])' E and: ' def test_1(session_fixture, function_fixture):' E and: '> raise AssertionError("fail")' E and: 'E AssertionError: fail' E and: '' E and: 'test_exception_matches_only_rerun_query.py:17: AssertionError' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'session setup' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '=========================== short test summary info ============================' E and: 'FAILED test_exception_matches_only_rerun_query.py::test_1 - AssertionError: fail' E and: '========================== 1 failed, 1 rerun in 0.02s ==========================' E remains unmatched: 'session teardown' /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0/tests/test_pytest_rerunfailures.py:1193: Failed ----------------------------- Captured stdout call ----------------------------- ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_matches_only_rerun_query0 plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 1 item test_exception_matches_only_rerun_query.py R [100%]F [100%] =================================== FAILURES =================================== ____________________________________ test_1 ____________________________________ session_fixture = 'session', function_fixture = 'function' @pytest.mark.flaky(reruns=1, only_rerun=["AssertionError"]) def test_1(session_fixture, function_fixture): > raise AssertionError("fail") E AssertionError: fail test_exception_matches_only_rerun_query.py:17: AssertionError ---------------------------- Captured stdout setup ----------------------------- session setup function setup --------------------------- Captured stdout teardown --------------------------- function teardown ---------------------------- Captured stdout setup ----------------------------- function setup --------------------------- Captured stdout teardown --------------------------- function teardown --------------------------- Captured stdout teardown --------------------------- function teardown =========================== short test summary info ============================ FAILED test_exception_matches_only_rerun_query.py::test_1 - AssertionError: fail ========================== 1 failed, 1 rerun in 0.02s ========================== ________________ test_exception_match_only_rerun_in_dual_query _________________ testdir = def test_exception_match_only_rerun_in_dual_query(testdir): testdir.makepyfile( """ import pytest @pytest.fixture(scope="session", autouse=True) def session_fixture(): print("session setup") yield "session" print("session teardown") @pytest.fixture(scope="function", autouse=True) def function_fixture(): print("function setup") yield "function" print("function teardown") @pytest.mark.flaky(reruns=1, rerun_except=["Not"], only_rerun=["Exception"]) def test_1(session_fixture, function_fixture): raise Exception("fail") """ ) result = testdir.runpytest() assert_outcomes(result, passed=0, failed=1, rerun=1) > result.stdout.fnmatch_lines("session teardown") E Failed: nomatch: 'session teardown' E and: '============================= test session starts ==============================' E and: 'platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0' E and: 'rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_match_only_rerun_in_dual_query0' E and: 'plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1' E and: 'collected 1 item' E and: '' E and: 'test_exception_match_only_rerun_in_dual_query.py R [100%]F [100%]' E and: '' E and: '=================================== FAILURES ===================================' E and: '____________________________________ test_1 ____________________________________' E and: '' E and: "session_fixture = 'session', function_fixture = 'function'" E and: '' E and: ' @pytest.mark.flaky(reruns=1, rerun_except=["Not"], only_rerun=["Exception"])' E and: ' def test_1(session_fixture, function_fixture):' E and: '> raise Exception("fail")' E and: 'E Exception: fail' E and: '' E and: 'test_exception_match_only_rerun_in_dual_query.py:17: Exception' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'session setup' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '---------------------------- Captured stdout setup -----------------------------' E and: 'function setup' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '--------------------------- Captured stdout teardown ---------------------------' E and: 'function teardown' E and: '=========================== short test summary info ============================' E and: 'FAILED test_exception_match_only_rerun_in_dual_query.py::test_1 - Exception: ...' E and: '========================== 1 failed, 1 rerun in 0.02s ==========================' E remains unmatched: 'session teardown' /home/tkloczko/rpmbuild/BUILD/pytest-rerunfailures-14.0/tests/test_pytest_rerunfailures.py:1274: Failed ----------------------------- Captured stdout call ----------------------------- ============================= test session starts ============================== platform linux -- Python 3.10.14, pytest-8.2.2, pluggy-1.5.0 rootdir: /tmp/pytest-of-tkloczko/pytest-15/test_exception_match_only_rerun_in_dual_query0 plugins: rerunfailures-14.0, forked-1.6.0, xdist-3.6.1 collected 1 item test_exception_match_only_rerun_in_dual_query.py R [100%]F [100%] =================================== FAILURES =================================== ____________________________________ test_1 ____________________________________ session_fixture = 'session', function_fixture = 'function' @pytest.mark.flaky(reruns=1, rerun_except=["Not"], only_rerun=["Exception"]) def test_1(session_fixture, function_fixture): > raise Exception("fail") E Exception: fail test_exception_match_only_rerun_in_dual_query.py:17: Exception ---------------------------- Captured stdout setup ----------------------------- session setup function setup --------------------------- Captured stdout teardown --------------------------- function teardown ---------------------------- Captured stdout setup ----------------------------- function setup --------------------------- Captured stdout teardown --------------------------- function teardown --------------------------- Captured stdout teardown --------------------------- function teardown =========================== short test summary info ============================ FAILED test_exception_match_only_rerun_in_dual_query.py::test_1 - Exception: ... ========================== 1 failed, 1 rerun in 0.02s ========================== =========================== short test summary info ============================ FAILED tests/test_pytest_rerunfailures.py::test_run_session_teardown_once_after_reruns FAILED tests/test_pytest_rerunfailures.py::test_exception_matches_rerun_except_query FAILED tests/test_pytest_rerunfailures.py::test_exception_not_match_rerun_except_query FAILED tests/test_pytest_rerunfailures.py::test_exception_matches_only_rerun_query FAILED tests/test_pytest_rerunfailures.py::test_exception_match_only_rerun_in_dual_query ======================== 5 failed, 104 passed in 9.72s ========================= ```
List of installed modules in build env: ```console Package Version ------------------ ----------- build 1.2.1 distro 1.9.0 exceptiongroup 1.1.3 execnet 2.1.1 importlib_metadata 7.1.0 iniconfig 2.0.0 installer 0.7.0 packaging 24.0 pluggy 1.5.0 py 1.11.0 pyproject_hooks 1.0.0 pytest 8.2.2 pytest-forked 1.6.0 pytest-xdist 3.6.1 python-dateutil 2.9.0.post0 setuptools 69.4.0 tokenize_rt 5.2.0 tomli 2.0.1 wheel 0.43.0 zipp 3.19.2 ```

Please let me know if you need more details or want me to perform some diagnostics.

icemac commented 3 months ago

This issue duplicates #267.