betamax crashes urllib3 v2.0.2

[kratsg]

It seems the newest version of urllib3 is breaking my tests using betamax with an attribute error

E                   AttributeError: 'MockHTTPResponse' object has no attribute 'close'

Bumping down to ~=1.26.15 works for me.

Click to expand

``` _________________________________ test_issue4 __________________________________ self = @contextmanager def _error_catcher(self) -> typing.Generator[None, None, None]: """ Catch low-level python exceptions, instead re-raising urllib3 variants, so that low-level exceptions are not leaked in the high-level api. On exit, release the connection back to the pool. """ clean_exit = False try: try: > yield clean_exit = False self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:705: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = , amt = 10240 def _raw_read( self, amt: int | None = None, ) -> bytes: """ Reads `amt` of bytes from the socket. """ if self._fp is None: return None # type: ignore[return-value] fp_closed = getattr(self._fp, "closed", False) with self._error_catcher(): data = self._fp_read(amt) if not fp_closed else b"" if amt is not None and amt != 0 and not data: # Platform-specific: Buggy versions of Python. # Close the connection when no data is returned # # This is redundant to what httplib/http.client _should_ # already do. However, versions of python released before # December 15, 2012 (http://bugs.python.org/issue16298) do # not properly close the connection in all cases. There is # no harm in redundantly calling close. self._fp.close() if ( self.enforce_content_length and self.length_remaining is not None and self.length_remaining != 0 ): # This is an edge case that httplib failed to cover due # to concerns of backward compatibility. We're # addressing it here to make sure IncompleteRead is # raised during streaming, so all calls with incorrect # Content-Length are caught. > raise IncompleteRead(self._fp_bytes_read, self.length_remaining) E urllib3.exceptions.IncompleteRead: IncompleteRead(1000 bytes read, 53104 more expected) amt = 10240 data = b'' fp_closed = False self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:830: IncompleteRead The above exception was the direct cause of the following exception: self = @contextmanager def _error_catcher(self) -> typing.Generator[None, None, None]: """ Catch low-level python exceptions, instead re-raising urllib3 variants, so that low-level exceptions are not leaked in the high-level api. On exit, release the connection back to the pool. """ clean_exit = False try: try: yield except SocketTimeout as e: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? if "read operation timed out" not in str(e): # SSL errors related to framing/MAC get wrapped and reraised here raise SSLError(e) from e raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except (HTTPException, OSError) as e: # This includes IncompleteRead. > raise ProtocolError(f"Connection broken: {e!r}", e) from e E urllib3.exceptions.ProtocolError: ('Connection broken: IncompleteRead(1000 bytes read, 53104 more expected)', IncompleteRead(1000 bytes read, 53104 more expected)) clean_exit = False self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:722: ProtocolError During handling of the above exception, another exception occurred: auth_session = def test_issue4(auth_session): with betamax.Betamax(auth_session).use_cassette("test_binaryData.test_issue4"): > response = auth_session.get( "uu-app-binarystore/getBinaryData", json={ "code": "fe4f85dd3740c53956c22bb4324065b8", "contentDisposition": "attachment", }, ) auth_session = tests/integration/test_binaryData.py:74: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/requests/sessions.py:600: in get return self.request("GET", url, **kwargs) kwargs = {'allow_redirects': True, 'json': {'code': 'fe4f85dd3740c53956c22bb4324065b8', 'contentDisposition': 'attachment'}} self = url = 'uu-app-binarystore/getBinaryData' src/itkdb/core.py:356: in request return super().request(method, url, *args, **kwargs) __class__ = args = () kwargs = {'allow_redirects': True, 'json': {'code': 'fe4f85dd3740c53956c22bb4324065b8', 'contentDisposition': 'attachment'}} method = 'GET' self = url = 'https://itkpd-test.unicorncollege.cz/uu-app-binarystore/getBinaryData' /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/requests/sessions.py:587: in request resp = self.send(prep, **send_kwargs) allow_redirects = True auth = None cert = None cookies = None data = None files = None headers = None hooks = None json = {'code': 'fe4f85dd3740c53956c22bb4324065b8', 'contentDisposition': 'attachment'} method = 'GET' params = None prep = proxies = {} req = self = send_kwargs = {'allow_redirects': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False, ...} settings = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'verify': True} stream = None timeout = None url = 'https://itkpd-test.unicorncollege.cz/uu-app-binarystore/getBinaryData' verify = None src/itkdb/core.py:344: in send response = super().send(request, **kwargs) __class__ = kwargs = {'allow_redirects': True, 'cert': None, 'proxies': OrderedDict(), 'stream': False, ...} request = self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/requests/sessions.py:745: in send r.content adapter = allow_redirects = True elapsed = 0.000541210[1745](https://gitlab.cern.ch/atlas-itk/sw/db/itkdb/-/jobs/29330758#L1745)605469 gen = history = [] hooks = {'response': []} kwargs = {'cert': None, 'proxies': OrderedDict(), 'stream': False, 'timeout': None, ...} r = request = self = start = 1683214677.3246856 stream = False /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/requests/models.py:899: in content self._content = b"".join(self.iter_content(CONTENT_CHUNK_SIZE)) or b"" self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/requests/models.py:816: in generate yield from self.raw.stream(chunk_size, decode_content=True) chunk_size = 10240 self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:935: in stream data = self.read(amt=amt, decode_content=decode_content) amt = 10240 decode_content = True self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:906: in read data = self._raw_read(amt) amt = 10240 cache_content = False data = b'VPA37913-W00221_Striptest_Segment_4_001.dat\nType: ATLAS18R1\nBatch: VPA37913\nWafer: 00221\nComponent: 20USES100002...0000001.47\n0010\t0000000.24\t0000038.15\t0000001.50\n0011\t0000000.24\t0000038.56\t0000001.49\n0012\t0000000.24\t0000' decode_content = True decoded_data = b'VPA37913-W00221_Striptest_Segment_4_001.dat\nType: ATLAS18R1\nBatch: VPA37913\nWafer: 00221\nComponent: 20USES100002...0000001.47\n0010\t0000000.24\t0000038.15\t0000001.50\n0011\t0000000.24\t0000038.56\t0000001.49\n0012\t0000000.24\t0000' flush_decoder = False self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:830: in _raw_read raise IncompleteRead(self._fp_bytes_read, self.length_remaining) amt = 10240 data = b'' fp_closed = False self = /usr/local/lib/python3.8/contextlib.py:131: in __exit__ self.gen.throw(type, value, traceback) self = traceback = type = value = IncompleteRead(1000 bytes read, 53104 more expected) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = @contextmanager def _error_catcher(self) -> typing.Generator[None, None, None]: """ Catch low-level python exceptions, instead re-raising urllib3 variants, so that low-level exceptions are not leaked in the high-level api. On exit, release the connection back to the pool. """ clean_exit = False try: try: yield except SocketTimeout as e: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? if "read operation timed out" not in str(e): # SSL errors related to framing/MAC get wrapped and reraised here raise SSLError(e) from e raise ReadTimeoutError(self._pool, None, "Read timed out.") from e # type: ignore[arg-type] except (HTTPException, OSError) as e: # This includes IncompleteRead. raise ProtocolError(f"Connection broken: {e!r}", e) from e # If no exception is thrown, we should avoid cleaning up # unnecessarily. clean_exit = True finally: # If we didn't terminate cleanly, we need to throw away our # connection. if not clean_exit: # The response may not be closed but we're not going to use it # anymore so close it now to ensure that the connection is # released back to the pool. if self._original_response: > self._original_response.close() E AttributeError: 'MockHTTPResponse' object has no attribute 'close' clean_exit = False self = /root/.local/share/hatch/env/virtual/itkdb/m8EZU4TW/dev.py3.8/lib/python3.8/site-packages/urllib3/response.py:735: AttributeError ```

[sigmavirus24]

Should be relatively easy enough to fix. I'd be happy to review a PR and try and get a release out if you'd like to send a fix.

[mgorny]

Ping. This is now blocking random packages in Gentoo.