skipping one of parametrized tests with `lazy_fixture` fails

[micuda]

Hi all,

Each test are run once for all environment in our test suite by default.

conftest.py

from dataclasses import dataclass
from enum        import auto, Flag

import pytest
from pytest_lazyfixture import is_lazy_fixture

class Environment(Flag):
  SLOVAK = auto()
  CZECH  = auto()
  ALL    = SLOVAK | CZECH

@dataclass
class TestItem:
  SUPPORTED_ENVIRONMENT_MARK_NAME = 'supported_environments'

  @classmethod
  def only_environment(cls, env):
    return getattr(pytest.mark, cls.SUPPORTED_ENVIRONMENT_MARK_NAME)(env)

  item: pytest.Function

  @property
  def current_environment(self):
    try:               current_environment = self.item.callspec.getparam(for_all_envs.__name__)
    except ValueError: current_environment = None
    else:
      if is_lazy_fixture(current_environment):
        current_environment = self.item._request.getfixturevalue(current_environment.name)

    return current_environment

  @property
  def supported_environments(self):
    return mark.args[0] \
      if   (mark := next(self.item.iter_markers(self.SUPPORTED_ENVIRONMENT_MARK_NAME), None)) \
      else Environment.ALL

@pytest.fixture(
  autouse = True,
  params  = [Environment.CZECH, Environment.SLOVAK],
)
def for_all_envs(request):
  ...

def pytest_runtest_setup(item):
  test_item = TestItem(item = item)

  if test_item.current_environment not in test_item.supported_environments:
    pytest.skip(f'cannot run on environment `{test_item.current_environment}`')

and the usage below:

test.py

from tests.conftest import Environment, TestItem

@TestItem.only_environment(Environment.CZECH)
def test_tests_skipping():
    ...

For now everything works well (output below):

collected 2 items

tests/test_tests_skipping.py::test_tests_skipping[Environment.CZECH] PASSED
tests/test_tests_skipping.py::test_tests_skipping[Environment.SLOVAK] SKIPPED (cannot run on environment `Environment.SLOVAK`)

Each environment needs custom setup and that's where the pytest_lazyfixture comes. I added two fixtures and changed parameter values of for_all_envs fixture.

conftest.py

@pytest.fixture
def czech_environment():
  # NOTE: do some env specific setup here
  return Environment.CZECH

@pytest.fixture
def slovak_environment():
  # NOTE: do some env specific setup here
  return Environment.SLOVAK

@pytest.fixture(
  autouse = True,
  params  = [
    pytest.param(lazy_fixture(czech_environment.__name__),  id = 'CZ'),
    pytest.param(lazy_fixture(slovak_environment.__name__), id = 'SK'),
  ]
)
def for_all_envs(request):
  return request.param

and now the output is:

collected 2 items

tests/test_tests_skipping.py::test_tests_skipping[cz] PASSED
tests/test_tests_skipping.py::test_tests_skipping[sk] SKIPPED (cannot run on environment `Environment.SLOVAK`)
tests/test_tests_skipping.py::test_tests_skipping[sk] ERROR

=========================================================================== ERRORS =================================================================================
________________________________________________________ ERROR at teardown of test_tests_skipping[sk] ______________________________________________________________
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:311: in from_call
    result: Optional[TResult] = func()
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:255: in <lambda>
    lambda: ihook(item=item, **kwds), when=when, reraise=reraise
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/pluggy/_hooks.py:265: in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/pluggy/_manager.py:80: in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:175: in pytest_runtest_teardown
    item.session._setupstate.teardown_exact(item, nextitem)
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:419: in teardown_exact
    self._teardown_towards(needed_collectors)
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:434: in _teardown_towards
    raise exc
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:427: in _teardown_towards
    self._pop_and_teardown()
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:387: in _pop_and_teardown
    self._teardown_with_finalization(colitem)
/home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:408: in _teardown_with_finalization
    assert colitem in self.stack
E   AssertionError

If the test is run for both env, everything is OK:

test.py

def test_tests_skipping():
    ...

output

collected 2 items

tests/test_tests_skipping.py::test_tests_skipping[cz] PASSED
tests/test_tests_skipping.py::test_tests_skipping[sk] PASSED

=========================================================================== 2 passed in 0.06s ===========================================================================

It seems there is missing colitem <Function test_tests_skipping[sk]> in the self.stack in /home/jcas/.pyenv/versions/3.9.6/envs/pytest_lazy_fixture_bug3.9.6/lib/python3.9/site-packages/_pytest/runner.py:408 (from the last frame).

I'm not sure what is wrong, because I don't know these pytest internals. Also I'm not sure if it is related to pytest-lazy-fixture.

Can anybody help me with this please?

[TvoroG]

Hi!

Try to use pytest_runtest_call instead of pytest_runtest_setup. Also you don't need pytest-lazy-fixture, just use request.getfixturevalue:

@dataclass
class TestItem:
    ...
    @property
    def current_environment(self):
        try:
            return self.item._request.getfixturevalue(for_all_envs.__name__)
        except ValueError:
            return None
   ...

@pytest.fixture(
    autouse = True,
    params  = [
        pytest.param(czech_environment.__name__,  id = 'CZ'),
        pytest.param(slovak_environment.__name__, id = 'SK'),
    ]
)
def for_all_envs(request):
    return request.getfixturevalue(request.param)

[micuda]

Thanks @TvoroG,

pytest_runtest_call does the work. =)