getsentry / responses

A utility for mocking out the Python Requests library.
Apache License 2.0
4.08k stars 347 forks source link

Bug when record the binary response to yaml #682

Open narugo1992 opened 8 months ago

narugo1992 commented 8 months ago

Describe the bug

When recording the binary response (e.g. image), the response is broken when added from file again.

Additional context

No response

Version of responses

0.23.3

Steps to Reproduce

Use this code to record the image's response from danbooru to danbooru.yaml

import logging

import requests
from responses import _recorder

logging.basicConfig(level=logging.DEBUG)

IMAGE_URL = 'https://cdn.donmai.us/original/e1/f8/e1f815ed0d489dddda86ac55815ce37f.jpg'

@_recorder.record(file_path='danbooru.yaml')
def main():
    resp = requests.get(IMAGE_URL)
    resp.raise_for_status()

if __name__ == '__main__':
    main()

and then use the following code to download this image both from original danbooru site (without mocking) and from mocked responses

import logging
import os

import requests
import responses
from tqdm.auto import tqdm

logging.basicConfig(level=logging.DEBUG)

def download_file(url, filename, expected_size: int = None, desc=None, **kwargs):
    session = requests.session()
    response = session.get(url, stream=True, allow_redirects=True, **kwargs)
    expected_size = expected_size or response.headers.get('Content-Length', None)
    expected_size = int(expected_size) if expected_size is not None else expected_size

    desc = desc or os.path.basename(filename)
    directory = os.path.dirname(filename)
    if directory:
        os.makedirs(directory, exist_ok=True)

    with open(filename, 'wb') as f:
        with tqdm(total=expected_size, unit='B', unit_scale=True, unit_divisor=1024, desc=desc) as pbar:
            for chunk in response.iter_content(chunk_size=1024):
                f.write(chunk)
                pbar.update(len(chunk))

    actual_size = os.path.getsize(filename)
    if expected_size is not None and actual_size != expected_size:
        os.remove(filename)
        raise requests.exceptions.HTTPError(f"Downloaded file is not of expected size, "
                                            f"{expected_size} expected but {actual_size} found.")

    return filename

responses._add_from_file('danbooru.yaml')

IMAGE_URL = 'https://cdn.donmai.us/original/e1/f8/e1f815ed0d489dddda86ac55815ce37f.jpg'

def test_danbooru_origin():
    download_file(IMAGE_URL, 'test_img_actual.jpg')

@responses.activate
def test_danbooru():
    download_file(IMAGE_URL, 'test_img.jpg')

if __name__ == '__main__':
    test_danbooru_origin()
    test_danbooru()

The you'll fine the test_img_actual.jpg is okay, its size is about 833KB.

But test_img.jpg is broken, its size is about 1.47MB.

Expected Result

The expected result of file test_img.jpg should be the same as test_img_origin.jpg.

Actual Result

And now it is broken, its size is obviously abnormal.