search

box / flaky

Plugin for nose or pytest that automatically reruns flaky tests.
Apache License 2.0
377 stars 60 forks source link

rerun_filter receives _pytest._code.Traceback instead of traceback.traceback #173

Open robertschweizer opened 3 years ago

robertschweizer commented 3 years ago

I was trying to inspect the current stack of the raised exception. For some reason, I'm receiving an stdlib traceback.traceback object in the PyCharm debugger, but pytest's internal _pytest._code.Traceback object when running without the debugger.

To reproduce:

Content of test_file.py:

import traceback
import flaky

def _print_tb(err, *_) -> bool:
    exc_type, exc, tb = err
    all_frames = traceback.extract_tb(tb)
    print([frame.name for frame in all_frames])
    return False

@flaky.flaky(max_runs=3, rerun_filter=_print_tb)
def test_test():
    raise ValueError()

Reproduced in the python:3.6 Docker image:

root@ca33330a0d53:/# pytest test_file.py
============================================================================================================= test session starts =============================================================================================================
platform linux -- Python 3.6.12, pytest-6.2.1, py-1.10.0, pluggy-0.13.1
rootdir: /
plugins: flaky-3.7.0
collected 1 item

test_file.py ['from_call', '<lambda>', '__call__', '_hookexec', '<lambda>', '_multicall', 'get_result', '_multicall', 'pytest_runtest_call', 'pytest_runtest_call', 'runtest', '__call__', '_hookexec', '<lambda>', '_multicall', 'get_result', '_multicall', 'pytest_pyfunc_call', 'test_test']
F                                                                                                                                                                                                                          [100%]
INTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/_pytest/main.py", line 269, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/_pytest/main.py", line 323, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/hooks.py", line 286, in __call__
INTERNALERROR>     return self._hookexec(self, self.get_hookimpls(), kwargs)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/manager.py", line 93, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/manager.py", line 87, in <lambda>
INTERNALERROR>     firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 208, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 80, in get_result
INTERNALERROR>     raise ex[1].with_traceback(ex[2])
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 187, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/_pytest/main.py", line 348, in pytest_runtestloop
INTERNALERROR>     item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/hooks.py", line 286, in __call__
INTERNALERROR>     return self._hookexec(self, self.get_hookimpls(), kwargs)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/manager.py", line 93, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/manager.py", line 87, in <lambda>
INTERNALERROR>     firstresult=hook.spec.opts.get("firstresult") if hook.spec else False,
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 208, in _multicall
INTERNALERROR>     return outcome.get_result()
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 80, in get_result
INTERNALERROR>     raise ex[1].with_traceback(ex[2])
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/pluggy/callers.py", line 187, in _multicall
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/flaky/flaky_pytest_plugin.py", line 110, in pytest_runtest_protocol
INTERNALERROR>     should_rerun = not skipped and self.add_failure(item, excinfo)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/flaky/flaky_pytest_plugin.py", line 336, in add_failure
INTERNALERROR>     return self._handle_test_error_or_failure(item, error)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/flaky/_flaky_plugin.py", line 190, in _handle_test_error_or_failure
INTERNALERROR>     if self._should_rerun_test(test, name, err):
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/flaky/_flaky_plugin.py", line 227, in _should_rerun_test
INTERNALERROR>     return rerun_filter(err, name, test, self)
INTERNALERROR>   File "/usr/local/lib/python3.6/site-packages/flaky/defaults.py", line 24, in __call__
INTERNALERROR>     return self._filter(*args, **kwargs)
INTERNALERROR>   File "/test_file.py", line 6, in _print_tb
INTERNALERROR>     all_frames = traceback.extract_tb(tb)
INTERNALERROR>   File "/usr/local/lib/python3.6/traceback.py", line 72, in extract_tb
INTERNALERROR>     return StackSummary.extract(walk_tb(tb), limit=limit)
INTERNALERROR>   File "/usr/local/lib/python3.6/traceback.py", line 345, in extract
INTERNALERROR>     for f, lineno in frame_gen:
INTERNALERROR>   File "/usr/local/lib/python3.6/traceback.py", line 310, in walk_tb
INTERNALERROR>     yield tb.tb_frame, tb.tb_lineno
INTERNALERROR> AttributeError: 'Traceback' object has no attribute 'tb_frame'

To work around this issue, you can get the traceback.traceback object:

from _pytest._code import Traceback

def _print_tb(err, *_) -> bool:
    exc_type, exc, tb = err
    if isinstance(tb, Traceback):
        tb = tb[0]._rawentry