jamielennox / requests-mock

Mocked responses for the requests library
https://requests-mock.readthedocs.io
Apache License 2.0
447 stars 71 forks source link

Async Dynamic response #154

Open BobCashStory opened 3 years ago

BobCashStory commented 3 years ago

i'm using request mock in my python package and i'm suck with a weird issue.

i need to use the mocker of other package instead of doing a http request but this one is async, so this doesn't work in request-mock, do you know any way to do it ?

Here are demo file: __init__.py

#empty 

conftest.py

import pytest

@pytest.fixture
def demo_sanic(caplog):
    caplog.set_level(logging.INFO)
    from .sanic_server import app
    yield app

@pytest.fixture
def test_sanic(loop, demo_sanic, sanic_client):
    return loop.run_until_complete(sanic_client(demo_sanic))

sanic_server.py

from sanic import Sanic
from sanic import response

app = Sanic(__name__)

@app.route("/test_get", methods=['GET'])
async def test_get(request):
    return response.json({"GET": True})

@app.route("/test_post", methods=['POST'])
async def test_post(request):
    return response.json({"POST": True})

sanic_client.py

import requests

def hightGetFunction():
    # stuff before
    req = requests.get("http://localhost:8000/test_get")
    req.raise_for_status()
    jsn = req.json()
    # stuff afet
    return jsn

def hightPostFunction():
    # stuff before
    req = requests.post("http://localhost:8000/test_post")
    req.raise_for_status()
    jsn = req.json()
    # stuff afet
    return jsn

test_sanic.py

from .sanic_client import hightGetFunction, hightPostFunction
import asyncio

def mock_job(requests_mock, test_sanic):

    def post_json(request, context):
        data = request.json
        coro = test_sanic.get(f"/", json=data)
        res = asyncio.ensure_future(coro).__await__()
        yield from res

    def get_json(request, context):
        data = {}
        coro = test_sanic.get(f"/", json=data)
        res = asyncio.ensure_future(coro).__await__()
        yield from res

    requests_mock.register_uri("GET", 'http://localhost:8000/test_get', json=get_json, status_code=200)
    requests_mock.register_uri("POST", 'http://localhost:8000/test_post', json=post_json, status_code=200)

async def test_get(requests_mock, test_sanic):
    mock_job(requests_mock, test_sanic)
    data = hightGetFunction()
    assert data == {"GET": True}

async def test_post(requests_mock, test_sanic):
    mock_job(requests_mock, test_sanic)
    data = hightPostFunction()
    assert data == {"POST": True}

TEST

with install: pip install pytest sanic requests-mock pytest-sanic requests asyncio Then i run : pytest and i get :

============================================================= test session starts =============================================================
platform darwin -- Python 3.8.5, pytest-5.4.3, py-1.9.0, pluggy-0.13.1
rootdir: /Users/martindonadieu/Documents/Projects.nosync/Naas/naas, inifile: pytest.ini
plugins: jupyter-server-1.0.3, requests-mock-1.8.0, flask-1.0.0, tornasync-0.6.0.post2, xdist-2.1.0, mock-3.3.1, cov-2.10.1, asyncio-0.14.0, sanic-1.6.1, forked-1.3.0
collected 2 items                                                                                                                             

test_sanic.py FF                                                                                                                        [100%]

================================================================== FAILURES ===================================================================
__________________________________________________________________ test_get ___________________________________________________________________

requests_mock = <requests_mock.mocker.Mocker object at 0x1202ab490>, test_sanic = <pytest_sanic.utils.TestClient object at 0x1202abb50>

    async def test_get(requests_mock, test_sanic):
        mock_job(requests_mock, test_sanic)
>       data = await hightGetFunction()

test_sanic.py:24: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sanic_client.py:5: in hightGetFunction
    req = requests.get("http://localhost:8000/test_get")
/usr/local/lib/python3.8/site-packages/requests/api.py:76: in get
    return request('get', url, params=params, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/api.py:61: in request
    return session.request(method=method, url=url, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:530: in request
    resp = self.send(prep, **send_kwargs)
/usr/local/lib/python3.8/site-packages/requests_mock/mocker.py:131: in _fake_send
    return _original_send(session, request, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:643: in send
    r = adapter.send(request, **kwargs)
/usr/local/lib/python3.8/site-packages/requests_mock/adapter.py:247: in send
    resp = matcher(request)
/usr/local/lib/python3.8/site-packages/requests_mock/adapter.py:227: in __call__
    return response_matcher.get_response(request)
/usr/local/lib/python3.8/site-packages/requests_mock/response.py:254: in get_response
    return create_response(request,
/usr/local/lib/python3.8/site-packages/requests_mock/response.py:166: in create_response
    text = jsonutils.dumps(json)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/__init__.py:231: in dumps
    return _default_encoder.encode(obj)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:199: in encode
    chunks = self.iterencode(o, _one_shot=True)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:257: in iterencode
    return _iterencode(o, 0)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <json.encoder.JSONEncoder object at 0x10bd7e520>, o = <generator object mock_job.<locals>.get_json at 0x1202a0a50>

    def default(self, o):
        """Implement this method in a subclass such that it returns
        a serializable object for ``o``, or calls the base implementation
        (to raise a ``TypeError``).

        For example, to support arbitrary iterators, you could
        implement default like this::

            def default(self, o):
                try:
                    iterable = iter(o)
                except TypeError:
                    pass
                else:
                    return list(iterable)
                # Let the base class default method raise the TypeError
                return JSONEncoder.default(self, o)

        """
>       raise TypeError(f'Object of type {o.__class__.__name__} '
                        f'is not JSON serializable')
E       TypeError: Object of type generator is not JSON serializable

/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:179: TypeError
------------------------------------------------------------ Captured stdout setup ------------------------------------------------------------
[2020-12-16 11:42:51 +0000] [87038] [INFO] Goin' Fast @ http://127.0.0.1:60802
------------------------------------------------------------- Captured log setup --------------------------------------------------------------
INFO     sanic.root:app.py:1386 Goin' Fast @ http://127.0.0.1:60802
__________________________________________________________________ test_post __________________________________________________________________

requests_mock = <requests_mock.mocker.Mocker object at 0x1204c89d0>, test_sanic = <pytest_sanic.utils.TestClient object at 0x1204c8160>

    async def test_post(requests_mock, test_sanic):
        mock_job(requests_mock, test_sanic)
>       data = await hightPostFunction()

test_sanic.py:29: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
sanic_client.py:13: in hightPostFunction
    req = requests.get("http://localhost:8000/test_get")
/usr/local/lib/python3.8/site-packages/requests/api.py:76: in get
    return request('get', url, params=params, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/api.py:61: in request
    return session.request(method=method, url=url, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:530: in request
    resp = self.send(prep, **send_kwargs)
/usr/local/lib/python3.8/site-packages/requests_mock/mocker.py:131: in _fake_send
    return _original_send(session, request, **kwargs)
/usr/local/lib/python3.8/site-packages/requests/sessions.py:643: in send
    r = adapter.send(request, **kwargs)
/usr/local/lib/python3.8/site-packages/requests_mock/adapter.py:247: in send
    resp = matcher(request)
/usr/local/lib/python3.8/site-packages/requests_mock/adapter.py:227: in __call__
    return response_matcher.get_response(request)
/usr/local/lib/python3.8/site-packages/requests_mock/response.py:254: in get_response
    return create_response(request,
/usr/local/lib/python3.8/site-packages/requests_mock/response.py:166: in create_response
    text = jsonutils.dumps(json)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/__init__.py:231: in dumps
    return _default_encoder.encode(obj)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:199: in encode
    chunks = self.iterencode(o, _one_shot=True)
/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:257: in iterencode
    return _iterencode(o, 0)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <json.encoder.JSONEncoder object at 0x10bd7e520>, o = <generator object mock_job.<locals>.get_json at 0x1204c1ac0>

    def default(self, o):
        """Implement this method in a subclass such that it returns
        a serializable object for ``o``, or calls the base implementation
        (to raise a ``TypeError``).

        For example, to support arbitrary iterators, you could
        implement default like this::

            def default(self, o):
                try:
                    iterable = iter(o)
                except TypeError:
                    pass
                else:
                    return list(iterable)
                # Let the base class default method raise the TypeError
                return JSONEncoder.default(self, o)

        """
>       raise TypeError(f'Object of type {o.__class__.__name__} '
                        f'is not JSON serializable')
E       TypeError: Object of type generator is not JSON serializable

/usr/local/Cellar/python@3.8/3.8.5/Frameworks/Python.framework/Versions/3.8/lib/python3.8/json/encoder.py:179: TypeError
------------------------------------------------------------ Captured stdout setup ------------------------------------------------------------
[2020-12-16 11:42:52 +0000] [87038] [INFO] Goin' Fast @ http://127.0.0.1:60803
------------------------------------------------------------- Captured log setup --------------------------------------------------------------
INFO     sanic.root:app.py:1386 Goin' Fast @ http://127.0.0.1:60803
============================================================== warnings summary ===============================================================
tests/tests/test_sanic.py::test_get
tests/tests/test_sanic.py::test_post
  /usr/local/lib/python3.8/site-packages/aiohttp/cookiejar.py:55: DeprecationWarning: The object should be created from async function
    super().__init__(loop=loop)

tests/tests/test_sanic.py::test_get
tests/tests/test_sanic.py::test_post
  /usr/local/lib/python3.8/site-packages/pytest_sanic/utils.py:191: DeprecationWarning: The object should be created from async function
    self._session = ClientSession(cookie_jar=cookie_jar,

tests/tests/test_sanic.py::test_get
tests/tests/test_sanic.py::test_post
  /usr/local/lib/python3.8/site-packages/aiohttp/connector.py:727: DeprecationWarning: The object should be created from async function
    super().__init__(keepalive_timeout=keepalive_timeout,

tests/tests/test_sanic.py::test_get
tests/tests/test_sanic.py::test_post
  /usr/local/lib/python3.8/site-packages/aiohttp/connector.py:736: DeprecationWarning: The object should be created from async function
    resolver = DefaultResolver(loop=self._loop)

-- Docs: https://docs.pytest.org/en/latest/warnings.html
=========================================================== short test summary info ===========================================================
FAILED test_sanic.py::test_get - TypeError: Object of type generator is not JSON serializable
FAILED test_sanic.py::test_post - TypeError: Object of type generator is not JSON serializable
======================================================== 2 failed, 8 warnings in 0.63s ========================================================

My question is: Does it's possible to have dynamic mock response with async in it ?

BobCashStory commented 3 years ago

okay i found a solution using https://github.com/miyakogi/syncer i replaced the mocker like that :

from .sanic_client import hightGetFunction, hightPostFunction
from syncer import sync
# import asyncio

def mock_job(requests_mock, test_sanic):

    def post_json(request, context):
        data = {}
        res = sync(test_sanic.post(f"/test_post", json=data))
        data_res = sync(res.json())
        return data_res

    def get_json(request, context):
        data = {}
        res = sync(test_sanic.get(f"/test_get", json=data))
        data_res = sync(res.json())
        return data_res

    requests_mock.register_uri("GET", 'http://localhost:8000/test_get', json=get_json, status_code=200)
    requests_mock.register_uri("POST", 'http://localhost:8000/test_post', json=post_json, status_code=200)

async def test_get(requests_mock, test_sanic):
    mock_job(requests_mock, test_sanic)
    data = hightGetFunction()
    assert data == {"GET": True}

async def test_post(requests_mock, test_sanic):
    mock_job(requests_mock, test_sanic)
    data = hightPostFunction()
    assert data == {"POST": True}
jamielennox commented 3 years ago

That's awesome.

No, I've never really used the async python features and this is the first time i've heard of sanic. Last I checked (a long time) the only requests async support was provided by some third party modules and there wasn't really a standard around that so it never progressed. Not that it matters here as it looks like the client is still using the standard sync, so you'd have to figure out that conversion somewhere.

Your example is interesting though, basically using requests-mock to build a bridge to an existing test api service.

If there's something specific there that would help i'd be happy to support it, but I don't know what it would be as requests is still fundamentally synchronous.

The only suggestion I'd have for a bridge like that is you can do it more generically (this is typed directly and won't compile, but i don't know what methods are available):

def callback(request, context):
    # fetch is some made up function to convert request.method into a GET/POST etc request
    res = sync(test_sanic.fetch(request.method, request.path, json=request.data))
    context.status_code = res.status_code
    context.headers.update(res.headers)
    return res.data

def is_sanic(request): 
    return request.netloc == 'localhost' and request.port == 8080

requests_mock.register_uri(requests_mock.ANY, requests_mock.ANY, additional_matcher=is_sanic, data=callback)

or it might be easier to just go the custom matching route and do all request checking/response creation in one go. You lose some history tracking, but depends if you need that on a matcher.

def custom_matcher(request): 
    if request.netloc == 'localhost' and request.port == 8080: 
        # call sanic, create requests.Response (can use requests_mock.create_response for help)
        return resp 

requests_mock.add_matcher(custom_matcher)