jayvdb / pypidb

PyPI client side database with SCM/VCS URLs
Apache License 2.0
13 stars 3 forks source link

Timeout on datarobot #64

Open jayvdb opened 4 years ago

jayvdb commented 4 years ago
___________________________________________________________________________ TestTopTail.test_package__<'datarobot'> ____________________________________________________________________________

self = <tests.test_top.TestTopTail testMethod=test_package__<'datarobot'>>, name = 'datarobot'

    @foreach(names)
    def test_package(self, name):
>       self._test_names([name])

tests/test_top.py:155: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
tests/utils.py:165: in _test_names
    url = self._get_scm(name)
tests/utils.py:98: in _get_scm
    url = self.converter.get_vcs(name)
pypidb/_pypi.py:559: in get_vcs
    url = self._get_vcs_links(rule, name, data, inputs)
pypidb/_pypi.py:340: in _get_vcs_links
    rv = self._accept_url(rule, name, url)
pypidb/_pypi.py:119: in _accept_url
    url = self._cleaner.get_root(url)
.tox/py38/lib/python3.8/site-packages/logger_helper/__init__.py:103: in wrapped_callable
    return_value = clbl(*args, **kwargs)
pypidb/_scm_url_cleaner.py:782: in get_root
    return self._get_root(url)
.tox/py38/lib/python3.8/site-packages/cachetools/decorators.py:22: in wrapper
    v = func(*args, **kwargs)
pypidb/_scm_url_cleaner.py:788: in _get_root
    rv = func(url)
pypidb/_scm_url_cleaner.py:291: in _follow_rtd_get_repo
    return _rtd_get_repo(url)
pypidb/_rtd.py:147: in get_repo
    url = _get_repo_v2_footer(subdomain, version, dot_com)
.tox/py38/lib/python3.8/site-packages/cachetools/decorators.py:22: in wrapper
    v = func(*args, **kwargs)
pypidb/_rtd.py:75: in _get_repo_v2_footer
    response = session.get(url)
.tox/py38/lib/python3.8/site-packages/requests/sessions.py:543: in get
    return self.request('GET', url, **kwargs)
.tox/py38/lib/python3.8/site-packages/requests/sessions.py:530: in request
    resp = self.send(prep, **send_kwargs)
.tox/py38/lib/python3.8/site-packages/requests/sessions.py:643: in send
    r = adapter.send(request, **kwargs)
pypidb/_cache.py:121: in send
    return super(ForceTimeoutHTTPAdapter, self).send(request, timeout=timeout, **kw)
pypidb/_adapters.py:59: in send
    resp = super(DomainListBlockAdapter, self).send(request, *args, **kwargs)
pypidb/_adapters.py:35: in send
    resp = super(IPBlockAdapter, self).send(request, *args, **kwargs)
pypidb/_adapters.py:91: in send
    resp = super(LoginBlockAdapter, self).send(request, *args, **kwargs)
pypidb/_adapters.py:110: in send
    resp = super(ContentTypeBlockAdapter, self).send(request, *args, **kwargs)
pypidb/_adapters.py:190: in send
    resp = super(HTTPSAdapter, self).send(request, *args, **kwargs)
.tox/py38/lib/python3.8/site-packages/https_everywhere/adapter.py:118: in send
    resp = super(RedirectAdapter, self).send(request, *args, **kwargs)
.tox/py38/lib/python3.8/site-packages/cachecontrol/adapter.py:53: in send
    resp = super(CacheControlAdapter, self).send(request, **kw)
.tox/py38/lib/python3.8/site-packages/requests/adapters.py:439: in send
    resp = conn.urlopen(
.tox/py38/lib/python3.8/site-packages/urllib3/connectionpool.py:665: in urlopen
    httplib_response = self._make_request(
.tox/py38/lib/python3.8/site-packages/urllib3/connectionpool.py:376: in _make_request
    self._validate_conn(conn)
.tox/py38/lib/python3.8/site-packages/urllib3/connectionpool.py:994: in _validate_conn
    conn.connect()
.tox/py38/lib/python3.8/site-packages/urllib3/connection.py:300: in connect
    conn = self._new_conn()
.tox/py38/lib/python3.8/site-packages/urllib3/connection.py:156: in _new_conn
    conn = connection.create_connection(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

address = ('readthedocs.com', 443), timeout = None, source_address = None, socket_options = [(6, 1, 1)]

    def create_connection(
        address,
        timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
        source_address=None,
        socket_options=None,
    ):
        """Connect to *address* and return the socket object.

        Convenience function.  Connect to *address* (a 2-tuple ``(host,
        port)``) and return the socket object.  Passing the optional
        *timeout* parameter will set the timeout on the socket instance
        before attempting to connect.  If no *timeout* is supplied, the
        global default timeout setting returned by :func:`getdefaulttimeout`
        is used.  If *source_address* is set it must be a tuple of (host, port)
        for the socket to bind as a source address before making the connection.
        An host of '' or port 0 tells the OS to use the default.
        """

        host, port = address
        if host.startswith("["):
            host = host.strip("[]")
        err = None

        # Using the value from allowed_gai_family() in the context of getaddrinfo lets
        # us select whether to work with IPv4 DNS records, IPv6 records, or both.
        # The original create_connection function always returns all records.
        family = allowed_gai_family()

        for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
            af, socktype, proto, canonname, sa = res
            sock = None
            try:
                sock = socket.socket(af, socktype, proto)

                # If provided, set socket level options before connecting.
                _set_socket_options(sock, socket_options)

                if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
                    sock.settimeout(timeout)
                if source_address:
                    sock.bind(source_address)
>               sock.connect(sa)
E               Failed: Timeout >100.0s

.tox/py38/lib/python3.8/site-packages/urllib3/util/connection.py:74: Failed
-------------------------------------------------------------------------------------- Captured log call ---------------------------------------------------------------------------------------
INFO     pypidb._pypi:_pypi.py:532 looking up datarobot
DEBUG    pypidb._adapters:_adapters.py:57 domain block of https://pypi.org/pypi/datarobot/json skipped
DEBUG    pypidb._adapters:_adapters.py:33 IPblock of https://pypi.org/pypi/datarobot/json skipped
DEBUG    https_everywhere.adapter:adapter.py:86 No implementation for get_redirect('https://pypi.org/pypi/datarobot/json')
DEBUG    https_everywhere.adapter:adapter.py:117 no redirection of https://pypi.org/pypi/datarobot/json occurred
DEBUG    cachecontrol.controller:controller.py:126 Looking up "https://pypi.org/pypi/datarobot/json" in the cache
DEBUG    cachecontrol.controller:controller.py:179 Current age based on date: 17023
DEBUG    cachecontrol.controller:controller.py:201 Freshness lifetime from expires: 431987
DEBUG    cachecontrol.controller:controller.py:219 The response is "fresh", returning cached response
DEBUG    cachecontrol.controller:controller.py:220 431987 > 17023
DEBUG    pypidb._pypi:_pypi.py:552 rule DefaultRule datarobot/datarobot = datarobot None {}
DEBUG    pypidb._pypi:_pypi.py:515 emails ['support@datarobot.com', 'info@datarobot.com']
DEBUG    pypidb._pypi:_pypi.py:307 queue [UrlSet({'https://pypi.org/project/datarobot/2.19.0/', 'http://datarobot.com', 'https://pypi.org/project/datarobot/'}), Text('This clien...'), Text('About data...'), Email('support@datarobot.com'), Email('info@datarobot.com')]
DEBUG    pypidb._pypi:_pypi.py:313 processing UrlSet: {'https://pypi.org/project/datarobot/2.19.0/', 'http://datarobot.com', 'https://pypi.org/project/datarobot/'}
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'https://pypi.org/project/datarobot/2.19.0/')
DEBUG    pypidb._scm_url_cleaner:__init__.py:177 Returned None from pypidb._scm_url_cleaner.SCMURLCleaner.get_root
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'http://datarobot.com')
DEBUG    pypidb._scm_url_cleaner:__init__.py:177 Returned None from pypidb._scm_url_cleaner.SCMURLCleaner.get_root
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'https://pypi.org/project/datarobot/')
DEBUG    pypidb._scm_url_cleaner:__init__.py:177 Returned None from pypidb._scm_url_cleaner.SCMURLCleaner.get_root
DEBUG    pypidb._pypi:_pypi.py:355 --none of this set: {'https://pypi.org/project/datarobot/2.19.0/', 'http://datarobot.com', 'https://pypi.org/project/datarobot/'}
DEBUG    pypidb._pypi:_pypi.py:357 queuing ['http://datarobot.com', 'https://pypi.org/project/datarobot/', 'https://pypi.org/project/datarobot/2.19.0/']
INFO     pypidb._pypi:_pypi.py:361 datarobot: from None added urls []
DEBUG    pypidb._pypi:_pypi.py:313 processing Text: summary
DEBUG    pypidb._pypi:_pypi.py:379 @@ ran <function _url_extractor_wrapper at 0x7f03e2f1b5e0> on text size 61 for 0 urls !!
DEBUG    pypidb._pypi:_pypi.py:384 extracted []
DEBUG    pypidb._pypi:_pypi.py:313 processing Text: description
DEBUG    pypidb._pypi:_pypi.py:379 @@ ran <function _url_extractor_wrapper at 0x7f03e2f1b5e0> on text size 1303 for 7 urls !!
DEBUG    pypidb._pypi:_pypi.py:384 extracted ['http://datarobot.com', 'https://app.datarobot.com/api/v2', 'https://datarobot-public-api-client.readthedocs-hosted.com', 'https://img.shields.io/pypi/pyversions/datarobot.svg', 'https://img.shields.io/pypi/status/datarobot.svg', 'https://img.shields.io/pypi/v/datarobot.svg', 'https://pypi.python.org/pypi/datarobot/']
DEBUG    pypidb._pypi:_pypi.py:313 processing Email: support@datarobot.com @ Unknown
INFO     pypidb._rules:_rules.py:872 email split: support @ datarobot.com
DEBUG    pypidb._pypi:_pypi.py:313 processing Email: info@datarobot.com @ Unknown
INFO     pypidb._rules:_rules.py:872 email split: info @ datarobot.com
DEBUG    pypidb._pypi:_pypi.py:313 processing Url: https://pypi.org/project/datarobot/2.19.0/ @ Unknown
DEBUG    pypidb._pypi:_pypi.py:423 reject rule https://pypi.org/project/datarobot/2.19.0/: False
INFO     pypidb._pypi:_pypi.py:447 r https://pypi.org/project/datarobot/2.19.0/
DEBUG    pypidb._pypi:_pypi.py:451 r Response.url https://pypi.org/project/datarobot/2.19.0/ elapsed 0:00:00.000078
DEBUG    pypidb._pypi:_pypi.py:456 r https://pypi.org/project/datarobot/2.19.0/ headers: {}
WARNING  pypidb._pypi:_pypi.py:459 https://pypi.org/project/datarobot/2.19.0/: 500 Server Error: HTTPS Everywhere for url: https://pypi.org/project/datarobot/2.19.0/
DEBUG    pypidb._pypi:_pypi.py:313 processing Url: http://datarobot.com @ Unknown
DEBUG    pypidb._pypi:_pypi.py:423 reject rule http://datarobot.com: False
INFO     pypidb._pypi:_pypi.py:447 r http://datarobot.com
DEBUG    pypidb._adapters:_adapters.py:57 domain block of http://datarobot.com/ skipped
DEBUG    pypidb._adapters:_adapters.py:33 IPblock of http://datarobot.com/ skipped
DEBUG    urllib3.connectionpool:connectionpool.py:221 Starting new HTTP connection (1): datarobot.com:80
DEBUG    urllib3.connectionpool:connectionpool.py:428 http://datarobot.com:80 "HEAD / HTTP/1.1" 301 0
DEBUG    pypidb._adapters:_adapters.py:220 head http://datarobot.com/ http://datarobot.com/ <Response [301]> {'Server': 'nginx', 'Date': 'Sun, 05 Apr 2020 08:56:10 GMT', 'Content-Type': 'text/html', 'Content-Length': '0', 'Location': 'http://www.datarobot.com/', 'Connection': 'Keep-Alive'} b''
INFO     https_everywhere.adapter:adapter.py:113 adapter redirecting http://datarobot.com/ to https://www.datarobot.com/
DEBUG    pypidb._adapters:_adapters.py:57 domain block of https://www.datarobot.com/ skipped
DEBUG    pypidb._adapters:_adapters.py:33 IPblock of https://www.datarobot.com/ skipped
DEBUG    https_everywhere.adapter:adapter.py:86 No implementation for get_redirect('https://www.datarobot.com/')
DEBUG    https_everywhere.adapter:adapter.py:117 no redirection of https://www.datarobot.com/ occurred
DEBUG    cachecontrol.controller:controller.py:126 Looking up "https://www.datarobot.com/" in the cache
DEBUG    cachecontrol.controller:controller.py:179 Current age based on date: 17019
DEBUG    cachecontrol.controller:controller.py:201 Freshness lifetime from expires: 431987
DEBUG    cachecontrol.controller:controller.py:219 The response is "fresh", returning cached response
DEBUG    cachecontrol.controller:controller.py:220 431987 > 17019
DEBUG    pypidb._pypi:_pypi.py:451 r Response.url https://www.datarobot.com/ elapsed 0:00:00.052358
DEBUG    pypidb._pypi:_pypi.py:456 r https://www.datarobot.com/ headers: {'Server': 'nginx', 'Date': 'Sun, 05 Apr 2020 04:12:18 GMT', 'Content-Type': 'text/html; charset=UTF-8', 'Connection': 'keep-alive', 'Keep-Alive': 'timeout=20', 'Vary': '', 'Link': '<https://www.datarobot.com/>; rel="alternate"; hreflang="en-US"', 'X-Powered-By': 'WP Engine', 'X-Cacheable': 'SHORT', 'cache-control': 'public', 'X-Cache': 'MISS', 'X-Cache-Group': 'normal', 'Content-Encoding': 'br', 'expires': 'Fri, 10 Apr 2020 04:12:05 -0000', 'Warning': '110 - Automatically cached for 5 days, 0:00:00. Response might be stale'}
DEBUG    pypidb._pypi:_pypi.py:313 processing Url: https://pypi.org/project/datarobot/ @ Unknown
DEBUG    pypidb._pypi:_pypi.py:423 reject rule https://pypi.org/project/datarobot/: False
INFO     pypidb._pypi:_pypi.py:447 r https://pypi.org/project/datarobot/
DEBUG    pypidb._pypi:_pypi.py:451 r Response.url https://pypi.org/project/datarobot/ elapsed 0:00:00.000066
DEBUG    pypidb._pypi:_pypi.py:456 r https://pypi.org/project/datarobot/ headers: {}
WARNING  pypidb._pypi:_pypi.py:459 https://pypi.org/project/datarobot/: 500 Server Error: HTTPS Everywhere for url: https://pypi.org/project/datarobot/
DEBUG    pypidb._pypi:_pypi.py:313 processing UrlSet: summary
DEBUG    pypidb._pypi:_pypi.py:313 processing UrlSet: description
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'https://pypi.python.org/pypi/datarobot/')
DEBUG    pypidb._scm_url_cleaner:__init__.py:177 Returned None from pypidb._scm_url_cleaner.SCMURLCleaner.get_root
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'https://img.shields.io/pypi/status/datarobot.svg')
DEBUG    pypidb._scm_url_cleaner:__init__.py:177 Returned None from pypidb._scm_url_cleaner.SCMURLCleaner.get_root
DEBUG    pypidb._scm_url_cleaner:__init__.py:157 Calling pypidb._scm_url_cleaner.SCMURLCleaner.get_root(self = <pypidb._scm_url_cleaner.SCMURLCleaner object at 0x7f03e11c3d90>, url = 'https://datarobot-public-api-client.readthedocs-hosted.com')
DEBUG    pypidb._adapters:_adapters.py:57 domain block of https://datarobot-public-api-client.readthedocs-hosted.com/ skipped
DEBUG    pypidb._adapters:_adapters.py:33 IPblock of https://datarobot-public-api-client.readthedocs-hosted.com/ skipped
DEBUG    https_everywhere.adapter:adapter.py:86 No implementation for get_redirect('https://datarobot-public-api-client.readthedocs-hosted.com/')
DEBUG    https_everywhere.adapter:adapter.py:117 no redirection of https://datarobot-public-api-client.readthedocs-hosted.com/ occurred
DEBUG    cachecontrol.controller:controller.py:126 Looking up "https://datarobot-public-api-client.readthedocs-hosted.com/" in the cache
DEBUG    cachecontrol.controller:controller.py:179 Current age based on date: 17018
DEBUG    cachecontrol.controller:controller.py:201 Freshness lifetime from expires: 431987
DEBUG    cachecontrol.controller:controller.py:219 The response is "fresh", returning cached response
DEBUG    cachecontrol.controller:controller.py:220 431987 > 17018
DEBUG    pypidb._adapters:_adapters.py:57 domain block of https://readthedocs.com/api/v2/footer_html/?format=json&project=datarobot-public-api-client&version=v2.19.0&page=index&docroot=/&source_suffix=.rst skipped
DEBUG    pypidb._adapters:_adapters.py:33 IPblock of https://readthedocs.com/api/v2/footer_html/?format=json&project=datarobot-public-api-client&version=v2.19.0&page=index&docroot=/&source_suffix=.rst skipped
DEBUG    https_everywhere.adapter:adapter.py:86 No implementation for get_redirect('https://readthedocs.com/api/v2/footer_html/?format=json&project=datarobot-public-api-client&version=v2.19.0&page=index&docroot=/&source_suffix=.rst')
DEBUG    https_everywhere.adapter:adapter.py:117 no redirection of https://readthedocs.com/api/v2/footer_html/?format=json&project=datarobot-public-api-client&version=v2.19.0&page=index&docroot=/&source_suffix=.rst occurred
DEBUG    cachecontrol.controller:controller.py:126 Looking up "https://readthedocs.com/api/v2/footer_html/?format=json&project=datarobot-public-api-client&version=v2.19.0&page=index&docroot=/&source_suffix=.rst" in the cache
DEBUG    cachecontrol.controller:controller.py:141 No cache entry available
DEBUG    urllib3.connectionpool:connectionpool.py:955 Starting new HTTPS connection (1): readthedocs.com:443
DEBUG    pypidb._scm_url_cleaner:__init__.py:198 Exception Failed occurred in pypidb._scm_url_cleaner.SCMURLCleaner.get_root, "Timeout >100.0s"