Sometimes when trying to download the IATI schema or registry data, a ConnectionError is raised. This error is currently not caught and so ends the whole process. We should add a retry around these calls to mitigate this.
Example from Feb 28th when downloading codelists:
Traceback (most recent call last):
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connectionpool.py", line 791, in urlopen
response = self._make_request(
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connectionpool.py", line 537, in _make_request
response = conn.getresponse()
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connection.py", line 461, in getresponse
httplib_response = super().getresponse()
File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
response.begin()
File "/usr/lib/python3.10/http/client.py", line 318, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/adapters.py", line 486, in send
resp = conn.urlopen(
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connectionpool.py", line 845, in urlopen
retries = retries.increment(
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/util/retry.py", line 470, in increment
raise reraise(type(error), error, _stacktrace)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/util/util.py", line 38, in reraise
raise value.with_traceback(tb)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connectionpool.py", line 791, in urlopen
response = self._make_request(
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connectionpool.py", line 537, in _make_request
response = conn.getresponse()
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/urllib3/connection.py", line 461, in getresponse
httplib_response = super().getresponse()
File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse
response.begin()
File "/usr/lib/python3.10/http/client.py", line 318, in begin
version, status, reason = self._read_status()
File "/usr/lib/python3.10/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/iatitables/iatitables/runner.py", line 6, in <module>
iatidata.run_all(processes=1)
File "/home/iatitables/iatitables/iatidata/__init__.py", line 1368, in run_all
process_registry(refresh=refresh, sample=sample, processes=processes)
File "/home/iatitables/iatitables/iatidata/__init__.py", line 335, in process_registry
save_all(sample=sample, parts=processes, refresh=refresh)
File "/home/iatitables/iatitables/iatidata/__init__.py", line 308, in save_all
get_standard(refresh)
File "/home/iatitables/iatitables/iatidata/__init__.py", line 101, in get_standard
iatikit.download.standard()
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/iatikit/utils/download.py", line 250, in standard
codelists()
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/iatikit/utils/download.py", line 187, in codelists
version_codelist = get_codelist(codelist_name, version)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/iatikit/utils/download.py", line 165, in get_codelist
version_codelist = requests.get(codelist_url).json()
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/api.py", line 73, in get
return request("get", url, params=params, **kwargs)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/api.py", line 59, in request
return session.request(method=method, url=url, **kwargs)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/sessions.py", line 589, in request
resp = self.send(prep, **send_kwargs)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/sessions.py", line 725, in send
history = [resp for resp in gen]
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/sessions.py", line 725, in <listcomp>
history = [resp for resp in gen]
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/sessions.py", line 266, in resolve_redirects
resp = self.send(
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/sessions.py", line 703, in send
r = adapter.send(request, **kwargs)
File "/home/iatitables/iatitables/.ve/lib/python3.10/site-packages/requests/adapters.py", line 501, in send
raise ConnectionError(err, request=request)
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
Sometimes when trying to download the IATI schema or registry data, a
ConnectionErroris raised. This error is currently not caught and so ends the whole process. We should add a retry around these calls to mitigate this.Example from Feb 28th when downloading codelists: