rawrgulmuffins commented 3 years ago

How do you use Sentry: We don't use Sentry but we do use responses

Problem Statement

Using a pytest fixture to build our test infrastructure (and reduce code reuse) has lead to weird errors in our test suite. We believe there is an issue with the registration process when it is done in a pytest fixture.

Minimal code Example

@pytest.fixture()
def register_responses():
    with responses.RequestsMock(assert_all_requests_are_fired=False) as response:
        response.add(
            method=responses.GET,
            url="http://example.com",
            content_type='application/json',
            json={"key": "value"},
            match_querystring=True,
        )
        response.add(
            method=responses.GET,
            url="http://different_example.com?hello=world",
            content_type='application/json',
            json={"key2": "value2"},
            match_querystring=True,
        )
    yield response

def test_request_params_multiple_urls_pytest_fixture(register_responses):
    resp = requests.get('http://example.com')
    assert resp.json() == {"key": "value"}
    resp = requests.get('http://different_example.com', params={"hello": "world"})
    assert resp.json() == {"key2": "value2"}
    assert responses.calls[1].request.params == {"hello": "world"}

Helper Function Example That Works

def helper_function():
    responses.add(
        method=responses.GET,
        url="http://example.com",
        content_type='application/json',
        json={"key": "value"},
        match_querystring=False,
    )
    responses.add(
        method=responses.GET,
        url="http://different_example.com?hello=world",
        content_type='application/json',
        json={"key2": "value2"},
        match_querystring=False,
    )

@responses.activate
def test_request_params_multiple_urls_helper_function():
    helper_function()

    resp = requests.get('http://example.com')
    assert resp.json() == {"key": "value"}
    resp = requests.get('http://different_example.com', params={"hello": "world"})
    assert resp.json() == {"key2": "value2"}
    assert responses.calls[1].request.params == {"hello": "world"}

Expected Result

The requests.get calls don't lead to mock not used errors.
with assert_all_requests_are_fired turned off there's no json stack trace from resp.json

Actual Result with `assert_all_requests_are_fired=True`

    def stop(self, allow_assert=True):
        self._patcher.stop()
        if not self.assert_all_requests_are_fired:
            return

        if not allow_assert:
            return

        not_called = [m for m in self._matches if m.call_count == 0]
        if not_called:
>           raise AssertionError(
                "Not all requests have been executed {0!r}".format(
                    [(match.method, match.url) for match in not_called]
                )
            )
E           AssertionError: Not all requests have been executed [('GET', 'http://example.com/'), ('GET', 'http://different_example.com/?hello=world')]

Actual Result with `assert_all_requests_are_fired=False`

register_responses = <responses.RequestsMock object at 0x7f036d8ff280>

    def test_request_params_multiple_urls_pytest_fixture(register_responses):
        resp = requests.get('http://example.com')
        breakpoint()
>       assert resp.json() == {"key": "value"}

responses_test.py:121:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/local/lib/python3.8/site-packages/requests/models.py:897: in json
    return complexjson.loads(self.text, **kwargs)
/usr/lib64/python3.8/json/__init__.py:357: in loads
    return _default_decoder.decode(s)
/usr/lib64/python3.8/json/decoder.py:337: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <json.decoder.JSONDecoder object at 0x7f036fcd1340>
s = '<!doctype html>\n<html>\n<head>\n    <title>Example Domain</title>\n\n    <meta charset="utf-8" />\n    <meta http-eq...on.</p>\n    <p><a href="https://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.

        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.

        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/usr/lib64/python3.8/json/decoder.py:355: JSONDecodeError

markstory commented 3 years ago

Have you tried using @pytest.yield_fixture? The pytest-responses uses this method and works well.

rawrgulmuffins commented 3 years ago

Thank you for the lightning quick response.

Switching to

@pytest.yield_fixture
def register_responses_2():
    with responses.RequestsMock() as response:
        response.add(
            method=responses.GET,
            url="http://example.com",
            content_type='application/json',
            json={"key": "value"},
            match_querystring=True,
        )
        response.add(
            method=responses.GET,
            url="http://different_example.com?hello=world",
            content_type='application/json',
            json={"key2": "value2"},
            match_querystring=True,
        )
    yield response

def test_request_params_multiple_urls_pytest_fixture_yield_fixture(register_responses_2):
    resp = requests.get('http://example.com')
    assert resp.json() == {"key": "value"}
    resp = requests.get('http://different_example.com', params={"hello": "world"})
    assert resp.json() == {"key2": "value2"}
    assert responses.calls[1].request.params == {"hello": "world"}

Still lead to

    @pytest.yield_fixture
    def register_responses_2():
        with responses.RequestsMock() as response:
            response.add(
                method=responses.GET,
                url="http://example.com",
                content_type='application/json',
                json={"key": "value"},
                match_querystring=True,
            )
>           response.add(
                method=responses.GET,
                url="http://different_example.com?hello=world",
                content_type='application/json',
                json={"key2": "value2"},
                match_querystring=True,
            )

responses_test.py:136:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/local/lib/python3.8/site-packages/responses.py:646: in __exit__
    self.stop(allow_assert=success)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <responses.RequestsMock object at 0x7fa2f7662280>, allow_assert = True

    def stop(self, allow_assert=True):
        self._patcher.stop()
        if not self.assert_all_requests_are_fired:
            return

        if not allow_assert:
            return

        not_called = [m for m in self._matches if m.call_count == 0]
        if not_called:
>           raise AssertionError(
                "Not all requests have been executed {0!r}".format(
                    [(match.method, match.url) for match in not_called]
                )
            )
E           AssertionError: Not all requests have been executed [('GET', 'http://example.com/'), ('GET', 'http://different_example.com/?hello=world')]

/usr/local/lib/python3.8/site-packages/responses.py:748: AssertionError

In my particular test scenario we have methods that make multiple calls to the same endpoint but with different query parameters which is why I have the two add calls but when I drop down to one add call I still get the same results.

    @pytest.yield_fixture
    def register_responses_3():
        with responses.RequestsMock() as response:
>           response.add(
                method=responses.GET,
                url="http://example.com",
                content_type='application/json',
                json={"key": "value"},
                match_querystring=True,
            )

responses_test.py:155:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/local/lib/python3.8/site-packages/responses.py:646: in __exit__
    self.stop(allow_assert=success)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <responses.RequestsMock object at 0x7f24c86a24c0>, allow_assert = True

    def stop(self, allow_assert=True):
        self._patcher.stop()
        if not self.assert_all_requests_are_fired:
            return

        if not allow_assert:
            return

        not_called = [m for m in self._matches if m.call_count == 0]
        if not_called:
>           raise AssertionError(
                "Not all requests have been executed {0!r}".format(
                    [(match.method, match.url) for match in not_called]
                )
            )
E           AssertionError: Not all requests have been executed [('GET', 'http://example.com/')]

/usr/local/lib/python3.8/site-packages/responses.py:748: AssertionError

ajhynes7 commented 3 years ago

Hi @rawrgulmuffins, I don't work on this project but I happened to come across this issue.

I'm wondering if your problem is that you have yield response outside of the with block. You could try indenting the yield line so it's inside the block.

import pytest
import responses

@pytest.fixture()
def register_responses():

    with responses.RequestsMock(assert_all_requests_are_fired=False) as response:
        # Other code...
        yield response

There's no need to use pytest.yield_fixture, because it's deprecated.

markstory commented 3 years ago

I think @ajhynes7 is right. Your yield should be within the with block.

rawrgulmuffins commented 3 years ago

Moving the yield statement under the with statement changed the error to a Connection refused error. Since I think the fixture is a valid use case according to the documentation I don't believe this is solved? Would love to be wrong. =P

Apologies to @ajhynes7 for the super delayed response on my end. I missed have accidentally marked this notification as done without responding.

Code Example

import responses
import requests

import pytest

@pytest.fixture()
def register_responses():
    with responses.RequestsMock() as response:
        responses.add(
            method=responses.GET,
            url="http://example.com",
            content_type='application/json',
            json={"key": "value"},
            match_querystring=True,
        )
        responses.add(
            method=responses.GET,
            url="http://different_example.com?hello=world",
            content_type='application/json',
            json={"key2": "value2"},
            match_querystring=True,
        )
        yield response

def test_request_params_multiple_urls_pytest_fixture(register_responses):
    resp = requests.get('http://example.com')
    assert resp.json() == {"key": "value"}
    resp = requests.get('http://different_example.com', params={"hello": "world"})
    assert resp.json() == {"key2": "value2"}
    assert responses.calls[1].request.params == {"hello": "world"}

Error

―――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――― test_request_params_multiple_urls_pytest_fixture ――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――――

register_responses = <responses.RequestsMock object at 0x7f10cc8ce550>

    def test_request_params_multiple_urls_pytest_fixture(register_responses):
>       resp = requests.get('http://example.com')

test_responses.py:119:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/local/lib/python3.8/site-packages/requests/api.py:75: in get
    return request('get', url, params=params, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/api.py:60: in request
    return session.request(method=method, url=url, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:533: in request
    resp = self.send(prep, **send_kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:646: in send
    r = adapter.send(request, **kwargs)
/usr/local/lib/python3.8/site-packages/responses.py:625: in unbound_on_send
    return self._on_request(adapter, request, *a, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <responses.RequestsMock object at 0x7f10cc8ce550>, adapter = <requests.adapters.HTTPAdapter object at 0x7f10cc8ce7f0>, request = <PreparedRequest [GET]>, kwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': None, ...}, match = None
resp_callback = None, error_msg = 'Connection refused: GET http://example.com/', response = ConnectionError('Connection refused: GET http://example.com/')

    def _on_request(self, adapter, request, **kwargs):
        match = self._find_match(request)
        resp_callback = self.response_callback

        if match is None:
            if request.url.startswith(self.passthru_prefixes):
                logger.info("request.allowed-passthru", extra={"url": request.url})
                return _real_send(adapter, request, **kwargs)

            error_msg = "Connection refused: {0} {1}".format(
                request.method, request.url
            )
            response = ConnectionError(error_msg)
            response.request = request

            self._calls.add(request, response)
            response = resp_callback(response) if resp_callback else response
>           raise response
E           requests.exceptions.ConnectionError: Connection refused: GET http://example.com/

/usr/local/lib/python3.8/site-packages/responses.py:600: ConnectionError

 api/test_responses.py::test_request_params_multiple_urls_pytest_fixture ⨯                                                                                                                                                                                     100% ██████████

Results (0.18s):
       1 failed
         - api/test_responses.py:118 test_request_params_multiple_urls_pytest_fixture

ajhynes7 commented 3 years ago

Hi @rawrgulmuffins, I tried out your code.

The sneaky problem was that you were calling add on responses (the library), not response (the context object).

You were also checking responses.calls, when it should be register_responses.calls.

The following code works for me:

import pytest
import requests
import responses

@pytest.fixture()
def register_responses():

    with responses.RequestsMock() as mock:

        mock.add(
            method=responses.GET,
            url="http://example.com",
            content_type="application/json",
            json={"key": "value"},
            match_querystring=True,
        )

        mock.add(
            method=responses.GET,
            url="http://different_example.com?hello=world",
            content_type="application/json",
            json={"key2": "value2"},
            match_querystring=True,
        )

        yield mock

def test_request_params_multiple_urls_pytest_fixture(register_responses):

    resp = requests.get("http://example.com")
    assert resp.json() == {"key": "value"}

    resp = requests.get("http://different_example.com", params={"hello": "world"})
    assert resp.json() == {"key2": "value2"}

    assert register_responses.calls[1].request.params == {"hello": "world"}

beliaev-maksim commented 2 years ago

@markstory I think the issue is explained and the solution is provided in the last message in this thread. Please close

getsentry / responses

Adding responses in a pytest fixture leads to unexpected test failures. #367

Problem Statement

Minimal code Example

Helper Function Example That Works

Expected Result

Actual Result with `assert_all_requests_are_fired=True`

Actual Result with `assert_all_requests_are_fired=False`

Code Example

Error

getsentry / responses

Adding responses in a pytest fixture leads to unexpected test failures. #367

Problem Statement

Minimal code Example

Helper Function Example That Works

Expected Result

Actual Result with assert_all_requests_are_fired=True

Actual Result with assert_all_requests_are_fired=False

Code Example

Error

Actual Result with `assert_all_requests_are_fired=True`

Actual Result with `assert_all_requests_are_fired=False`