kpi_008: RemoteDisconnected: Remote end closed connection without response

[maaikelimper]

When checking links for kpi_008 pywcmp occassionally crashes due to not getting an response. For example for identifier: 'urn:x-wmo:md:int.wmo.wis::CUIS01LLBD'

I also note that the code does not specify a specific time-out error: response = urlopen(url, context=context) Could we add a fixed timeout response = urlopen(url, context=context,timeout=timeout) (5 seconds? 30 seconds?) to wait to validate a link and ensure the code catches the RemoteDisconnected-Exception ?

/usr/local/lib/python3.7/site-packages/pywcmp-0.4.dev0-py3.7.egg/pywcmp/kpi.py in kpi_008(self)
    591         for link in links:
    592             LOGGER.debug(f'checking: {link}')
--> 593             result = check_url(link, False)
    594             total += 2
    595             if result['accessible']:

/usr/local/lib/python3.7/site-packages/pywcmp-0.4.dev0-py3.7.egg/pywcmp/util.py in check_url(url, check_ssl)
    308             result['ssl'] = False
    309             context = ssl._create_unverified_context()
--> 310             response = urlopen(url, context=context)
    311         else:
    312             response = urlopen(url)

/usr/local/lib/python3.7/urllib/request.py in urlopen(url, data, timeout, cafile, capath, cadefault, context)
    220     else:
    221         opener = _opener
--> 222     return opener.open(url, data, timeout)
    223 
    224 def install_opener(opener):

/usr/local/lib/python3.7/urllib/request.py in open(self, fullurl, data, timeout)
    523             req = meth(req)
    524 
--> 525         response = self._open(req, data)
    526 
    527         # post-process response

/usr/local/lib/python3.7/urllib/request.py in _open(self, req, data)
    541         protocol = req.type
    542         result = self._call_chain(self.handle_open, protocol, protocol +
--> 543                                   '_open', req)
    544         if result:
    545             return result

/usr/local/lib/python3.7/urllib/request.py in _call_chain(self, chain, kind, meth_name, *args)
    501         for handler in handlers:
    502             func = getattr(handler, meth_name)
--> 503             result = func(*args)
    504             if result is not None:
    505                 return result

/usr/local/lib/python3.7/urllib/request.py in http_open(self, req)
   1376 
   1377     def http_open(self, req):
-> 1378         return self.do_open(http.client.HTTPConnection, req)
   1379 
   1380     http_request = AbstractHTTPHandler.do_request_

/usr/local/lib/python3.7/urllib/request.py in do_open(self, http_class, req, **http_conn_args)
   1351             except OSError as err: # timeout error
   1352                 raise URLError(err)
-> 1353             r = h.getresponse()
   1354         except:
   1355             h.close()

/usr/local/lib/python3.7/http/client.py in getresponse(self)
   1371         try:
   1372             try:
-> 1373                 response.begin()
   1374             except ConnectionError:
   1375                 self.close()

/usr/local/lib/python3.7/http/client.py in begin(self)
    317         # read until we get a non-100 response
    318         while True:
--> 319             version, status, reason = self._read_status()
    320             if status != CONTINUE:
    321                 break

/usr/local/lib/python3.7/http/client.py in _read_status(self)
    286             # Presumably, the server closed the connection before
    287             # sending a valid response.
--> 288             raise RemoteDisconnected("Remote end closed connection without"
    289                                      " response")
    290         try:

RemoteDisconnected: Remote end closed connection without response

[maaikelimper]

https://oai.dwd.de/oai/provider?verb=GetRecord&identifier=urn:x-wmo:md:int.wmo.wis::CUIS01LLBD&metadataPrefix=iso19139

[tomkralidis]

PR in #69