search

mozilla / bugbot

A Mozilla release management tool to send reminders to Firefox developers and improve Bugzilla metadata
BSD 3-Clause "New" or "Revised" License
41 stars 68 forks source link

timed out: whattrainisitnow.com #1932

Closed suhaibmujahid closed 1 year ago

suhaibmujahid commented 1 year ago 
2023-03-13 12:07:58,186 - ERROR - Tool no_severity
Traceback (most recent call last):
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 597, in urlopen
    self._prepare_proxy(conn)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 807, in _prepare_proxy
    conn.connect()
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connection.py", line 324, in connect
    self._tunnel()
  File "/usr/local/lib/python3.7/http/client.py", line 906, in _tunnel
    (version, code, message) = response._read_status()
  File "/usr/local/lib/python3.7/http/client.py", line 257, in _read_status
    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
  File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
    return self._sock.recv_into(b)
socket.timeout: timed out

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/adapters.py", line 449, in send
    timeout=timeout
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 641, in urlopen
    _stacktrace=sys.exc_info()[2])
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/util/retry.py", line 399, in increment
    raise MaxRetryError(_pool, url, error or ResponseError(cause))
urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='whattrainisitnow.com', port=443): Max retries exceeded with url: /api/release/schedule/?version=beta (Caused by ProxyError('Cannot connect to proxy.', timeout('timed out')))

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 839, in run
    self.send_email(date=date)
  File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 762, in send_email
    data = self.get_email_data(date)
  File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 680, in get_email_data
    bugs = self.get_bugs(date=date)
  File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 454, in get_bugs
    timeout=self.get_config("bz_query_timeout"),
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 176, in wait
    self.wait_bugs()
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 187, in wait_bugs
    r.result()
  File "/usr/local/lib/python3.7/concurrent/futures/_base.py", line 432, in result
    return self.__get_result()
  File "/usr/local/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result
    raise self._exception
  File "/usr/local/lib/python3.7/concurrent/futures/thread.py", line 57, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 653, in send
    r = dispatch_hook('response', hooks, r, **kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/hooks.py", line 31, in dispatch_hook
    _hook_data = hook(hook_data, **kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 552, in __bugs_cb
    self.bughandler.handle(bug)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/handler.py", line 15, in handle
    self.handler(*args)
  File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 367, in bughandler
    bug = self.set_people_to_nag(bug, res)
  File "/home/rm_bot/relman-auto-nag/auto_nag/scripts/workflow/no_severity.py", line 100, in set_people_to_nag
    if not self.filter_bug(priority):
  File "/home/rm_bot/relman-auto-nag/auto_nag/nag_me.py", line 47, in filter_bug
    days = (utils.get_next_release_date() - self.nag_date).days
  File "/home/rm_bot/relman-auto-nag/auto_nag/utils.py", line 221, in get_next_release_date
    schedule = FirefoxTrains().get_release_schedule("beta")
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/fx_trains.py", line 33, in get_release_schedule
    return self.__get(api_path)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/fx_trains.py", line 17, in __get
    resp = requests.get(self.URL + path, timeout=10)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/api.py", line 75, in get
    return request('get', url, params=params, **kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/api.py", line 60, in request
    return session.request(method=method, url=url, **kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 533, in request
    resp = self.send(prep, **send_kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 668, in send
    history = [resp for resp in gen] if allow_redirects else []
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 668, in <listcomp>
    history = [resp for resp in gen] if allow_redirects else []
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 247, in resolve_redirects
    **adapter_kwargs
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 646, in send
    r = adapter.send(request, **kwargs)
  File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/adapters.py", line 510, in send
    raise ProxyError(e, request=request)
requests.exceptions.ProxyError: HTTPSConnectionPool(host='whattrainisitnow.com', port=443): Max retries exceeded with url: /api/release/schedule/?version=beta (Caused by ProxyError('Cannot connect to proxy.', timeout('timed out')))
pascalchevrel commented 1 year ago

I just did a load test with ab and 1000 requests and all requests succeeded:

ab -n 1000 -c 10 https://whattrainisitnow.com/api/release/schedule/?version=beta
This is ApacheBench, Version 2.3 <$Revision: 1901567 $>
Copyright 1996 Adam Twiss, Zeus Technology Ltd, http://www.zeustech.net/
Licensed to The Apache Software Foundation, http://www.apache.org/

Benchmarking whattrainisitnow.com (be patient)
Completed 100 requests
Completed 200 requests
Completed 300 requests
Completed 400 requests
Completed 500 requests
Completed 600 requests
Completed 700 requests
Completed 800 requests
Completed 900 requests
Completed 1000 requests
Finished 1000 requests

Server Software:        Apache
Server Hostname:        whattrainisitnow.com
Server Port:            443
SSL/TLS Protocol:       TLSv1.3,TLS_AES_128_GCM_SHA256,2048,128
Server Temp Key:        X25519 253 bits
TLS Server Name:        whattrainisitnow.com

Document Path:          /api/release/schedule/?version=beta
Document Length:        759 bytes

Concurrency Level:      10
Time taken for tests:   40.007 seconds
Complete requests:      1000
Failed requests:        0
Total transferred:      1417000 bytes
HTML transferred:       759000 bytes
Requests per second:    25.00 [#/sec] (mean)
Time per request:       400.069 [ms] (mean)
Time per request:       40.007 [ms] (mean, across all concurrent requests)
Transfer rate:          34.59 [Kbytes/sec] received

Connection Times (ms)
              min  mean[+/-sd] median   max
Connect:      171  180   5.2    181     208
Processing:   205  214   7.0    215     352
Waiting:      205  214   7.0    215     352
Total:        377  394  10.6    396     528

Percentage of the requests served within a certain time (ms)
  50%    396
  66%    400
  75%    402
  80%    403
  90%    405
  95%    407
  98%    411
  99%    419
 100%    528 (longest request)

The error message talks about a proxy error, are the autonag scripts behind a proxy? Remove the proxy? Increase the timeout before failing?

Googling this error message, I saw a stackoverflow comment with a similar error message stating that updating the requests library fixed it for them, I don't know which version you use but maybe worth a shot.

suhaibmujahid commented 1 year ago

Thank you, Pascal!

The error message talks about a proxy error, are the autonag scripts behind a proxy?

Yes

Remove the proxy?

It is an intermittent error

Increase the timeout before failing?

I will try to do that as a workaround if other solutions fail to solve the issue.

Googling this error message, I saw a stackoverflow comment with a similar error message stating that updating the requests library fixed it for them, I don't know which version you use but maybe worth a shot.

Currently we use v2.22.0, I update it to the latest version in #1935; let's hope it will fix it :)

suhaibmujahid commented 1 year ago
Again the same timeout issue ``` 2023-03-23 12:03:35,287 - ERROR - Tool tracking Traceback (most recent call last): File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 597, in urlopen self._prepare_proxy(conn) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 807, in _prepare_proxy conn.connect() File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connection.py", line 324, in connect self._tunnel() File "/usr/local/lib/python3.7/http/client.py", line 906, in _tunnel (version, code, message) = response._read_status() File "/usr/local/lib/python3.7/http/client.py", line 257, in _read_status line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") File "/usr/local/lib/python3.7/socket.py", line 589, in readinto return self._sock.recv_into(b) socket.timeout: timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/adapters.py", line 449, in send timeout=timeout File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/connectionpool.py", line 641, in urlopen _stacktrace=sys.exc_info()[2]) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/urllib3/util/retry.py", line 399, in increment raise MaxRetryError(_pool, url, error or ResponseError(cause)) urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='whattrainisitnow.com', port=443): Max retries exceeded with url: /api/release/schedule/?version=beta (Caused by ProxyError('Cannot connect to proxy.', timeout('timed out'))) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 839, in run self.send_email(date=date) File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 762, in send_email data = self.get_email_data(date) File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 680, in get_email_data bugs = self.get_bugs(date=date) File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 454, in get_bugs timeout=self.get_config("bz_query_timeout"), File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 176, in wait self.wait_bugs() File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 187, in wait_bugs r.result() File "/usr/local/lib/python3.7/concurrent/futures/_base.py", line 432, in result return self.__get_result() File "/usr/local/lib/python3.7/concurrent/futures/_base.py", line 384, in __get_result raise self._exception File "/usr/local/lib/python3.7/concurrent/futures/thread.py", line 57, in run result = self.fn(*self.args, **self.kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 533, in request resp = self.send(prep, **send_kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 653, in send r = dispatch_hook('response', hooks, r, **kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/hooks.py", line 31, in dispatch_hook _hook_data = hook(hook_data, **kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/bugzilla.py", line 552, in __bugs_cb self.bughandler.handle(bug) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/handler.py", line 15, in handle self.handler(*args) File "/home/rm_bot/relman-auto-nag/auto_nag/bzcleaner.py", line 367, in bughandler bug = self.set_people_to_nag(bug, res) File "/home/rm_bot/relman-auto-nag/auto_nag/scripts/tracking.py", line 72, in set_people_to_nag if not self.filter_bug(priority): File "/home/rm_bot/relman-auto-nag/auto_nag/nag_me.py", line 47, in filter_bug days = (utils.get_next_release_date() - self.nag_date).days File "/home/rm_bot/relman-auto-nag/auto_nag/utils.py", line 221, in get_next_release_date schedule = FirefoxTrains().get_release_schedule("beta") File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/fx_trains.py", line 33, in get_release_schedule return self.__get(api_path) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/libmozdata/fx_trains.py", line 17, in __get resp = requests.get(self.URL + path, timeout=10) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/api.py", line 75, in get return request('get', url, params=params, **kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/api.py", line 60, in request return session.request(method=method, url=url, **kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 533, in request resp = self.send(prep, **send_kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 668, in send history = [resp for resp in gen] if allow_redirects else [] File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 668, in history = [resp for resp in gen] if allow_redirects else [] File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 247, in resolve_redirects **adapter_kwargs File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/sessions.py", line 646, in send r = adapter.send(request, **kwargs) File "/home/rm_bot/relman-auto-nag/venv/lib/python3.7/site-packages/requests/adapters.py", line 510, in send raise ProxyError(e, request=request) requests.exceptions.ProxyError: HTTPSConnectionPool(host='whattrainisitnow.com', port=443): Max retries exceeded with url: /api/release/schedule/?version=beta (Caused by ProxyError('Cannot connect to proxy.', timeout('timed out'))) ```