search

J535D165 / cbsodata

Unofficial Statistics Netherlands (CBS) open data API client for Python
http://cbsodata.readthedocs.io/
MIT License
42 stars 17 forks source link

ConnectionError: ('Connection aborted.', OSError("(54, 'ECONNRESET')",)) #5

Closed J535D165 closed 4 years ago

J535D165 commented 6 years ago 
In [1]: In [1]: import cbsodata
   ...: In [2]: cbsodata.CBSOPENDATA = "dataderden.cbs.nl"
   ...: In [3]: cbsodata.get_data('47015NED')
   ...: 
---------------------------------------------------------------------------
OSError                                   Traceback (most recent call last)
~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    599                                                   body=body, headers=headers,
--> 600                                                   chunked=chunked)
    601 

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    383                     # otherwise it looks like a programming error was the cause.
--> 384                     six.raise_from(e, None)
    385         except (SocketTimeout, BaseSSLError, SocketError) as e:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/packages/six.py in raise_from(value, from_value)

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    379                 try:
--> 380                     httplib_response = conn.getresponse()
    381                 except Exception as e:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in getresponse(self)
   1330             try:
-> 1331                 response.begin()
   1332             except ConnectionError:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in begin(self)
    296         while True:
--> 297             version, status, reason = self._read_status()
    298             if status != CONTINUE:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in _read_status(self)
    257     def _read_status(self):
--> 258         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    259         if len(line) > _MAXLINE:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/socket.py in readinto(self, b)
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py in recv_into(self, *args, **kwargs)
    289             else:
--> 290                 raise SocketError(str(e))
    291         except OpenSSL.SSL.ZeroReturnError as e:

OSError: (54, 'ECONNRESET')

During handling of the above exception, another exception occurred:

ProtocolError                             Traceback (most recent call last)
~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    444                     retries=self.max_retries,
--> 445                     timeout=timeout
    446                 )

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    637             retries = retries.increment(method, url, error=e, _pool=self,
--> 638                                         _stacktrace=sys.exc_info()[2])
    639             retries.sleep()

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    366             if read is False or not self._is_method_retryable(method):
--> 367                 raise six.reraise(type(error), error, _stacktrace)
    368             elif read is not None:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/packages/six.py in reraise(tp, value, tb)
    684         if value.__traceback__ is not tb:
--> 685             raise value.with_traceback(tb)
    686         raise value

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    599                                                   body=body, headers=headers,
--> 600                                                   chunked=chunked)
    601 

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    383                     # otherwise it looks like a programming error was the cause.
--> 384                     six.raise_from(e, None)
    385         except (SocketTimeout, BaseSSLError, SocketError) as e:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/packages/six.py in raise_from(value, from_value)

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    379                 try:
--> 380                     httplib_response = conn.getresponse()
    381                 except Exception as e:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in getresponse(self)
   1330             try:
-> 1331                 response.begin()
   1332             except ConnectionError:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in begin(self)
    296         while True:
--> 297             version, status, reason = self._read_status()
    298             if status != CONTINUE:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/http/client.py in _read_status(self)
    257     def _read_status(self):
--> 258         line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
    259         if len(line) > _MAXLINE:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/socket.py in readinto(self, b)
    585             try:
--> 586                 return self._sock.recv_into(b)
    587             except timeout:

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/urllib3/contrib/pyopenssl.py in recv_into(self, *args, **kwargs)
    289             else:
--> 290                 raise SocketError(str(e))
    291         except OpenSSL.SSL.ZeroReturnError as e:

ProtocolError: ('Connection aborted.', OSError("(54, 'ECONNRESET')",))

During handling of the above exception, another exception occurred:

ConnectionError                           Traceback (most recent call last)
<ipython-input-1-b54313817458> in <module>()
      1 import cbsodata
      2 cbsodata.CBSOPENDATA = "dataderden.cbs.nl"
----> 3 cbsodata.get_data('47015NED')

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/cbsodata.py in get_data(table_id, dir, typed, select, filters)
    277 
    278     metadata = download_data(table_id, dir=dir, typed=typed,
--> 279                              select=select, filters=filters)
    280 
    281     if "TypedDataSet" in metadata.keys():

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/cbsodata.py in download_data(table_id, dir, typed, select, filters)
    171         if table_name in ["TypedDataSet", "UntypedDataSet"]:
    172             metadata = _download_metadata(table_id, table_name,
--> 173                                           select=select, filters=filters)
    174         else:
    175             metadata = _download_metadata(table_id, table_name)

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/cbsodata.py in _download_metadata(table_id, metadata_name, select, filters)
     78     while (url is not None):
     79 
---> 80         r = requests.get(url, params=params)
     81 
     82         res = r.json(encoding='utf-8')

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/api.py in get(url, params, **kwargs)
     70 
     71     kwargs.setdefault('allow_redirects', True)
---> 72     return request('get', url, params=params, **kwargs)
     73 
     74 

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/api.py in request(method, url, **kwargs)
     56     # cases, and look like a memory leak in others.
     57     with sessions.Session() as session:
---> 58         return session.request(method=method, url=url, **kwargs)
     59 
     60 

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    510         }
    511         send_kwargs.update(settings)
--> 512         resp = self.send(prep, **send_kwargs)
    513 
    514         return resp

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/sessions.py in send(self, request, **kwargs)
    620 
    621         # Send the request
--> 622         r = adapter.send(request, **kwargs)
    623 
    624         # Total elapsed time of the request (approximately)

~/.pyenv/versions/anaconda3-5.0.1/lib/python3.6/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    493 
    494         except (ProtocolError, socket.error) as err:
--> 495             raise ConnectionError(err, request=request)
    496 
    497         except MaxRetryError as e:

ConnectionError: ('Connection aborted.', OSError("(54, 'ECONNRESET')",))
g0ldm45k commented 5 years ago

I'm also seeing this problem when trying with 37325. I am guessing it is a limitation from the CBS's end because the dataset is larger than their limit of 10.000 cells. (https://www.cbs.nl/nl-nl/onze-diensten/open-data/databank-cbs-statline-als-open-data). 47015NED is 14.2 million cells large. (https://data.politie.nl/portal.html?_la=nl&_catalog=Politie&tableId=47015NED&_theme=79).

J535D165 commented 5 years ago

Thanks for reporting. This is a problem om the CBS server side. See also https://github.com/edwindj/cbsodataR/issues/17. I'm not affiliated to CBS, so maybe @edwindj knows more about the status of these issues.