search

botify-labs / simpleflow

Python library for dataflow programming.
https://botify-labs.github.com/simpleflow/
MIT License
68 stars 24 forks source link

boto3: Read timeouts in pollers #432

Closed ybastide closed 11 months ago

ybastide commented 11 months ago

Pollers raise every 2 minutes on the boto3 rc1.

Details Decider: ``` 2023-09-26T19:03:25 INFO [process=MainProcess, pid=18010]: starting DeciderPoller(workflow=analysis) on domain XXX Process Process-44: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 449, in _make_request six.raise_from(e, None) File "", line 3, in raise_from File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 444, in _make_request httplib_response = conn.getresponse() File "/usr/lib/python3.7/http/client.py", line 1348, in getresponse response.begin() File "/usr/lib/python3.7/http/client.py", line 315, in begin version, status, reason = self._read_status() File "/usr/lib/python3.7/http/client.py", line 276, in _read_status line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") File "/usr/lib/python3.7/socket.py", line 589, in readinto return self._sock.recv_into(b) File "/usr/lib/python3.7/ssl.py", line 1071, in recv_into return self.read(nbytes, buffer) File "/usr/lib/python3.7/ssl.py", line 929, in read return self._sslobj.read(len, buffer) socket.timeout: The read operation timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/botocore/httpsession.py", line 474, in send chunked=self._chunked(request.headers), File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 788, in urlopen method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] File "/usr/local/lib/python3.7/dist-packages/urllib3/util/retry.py", line 525, in increment raise six.reraise(type(error), error, _stacktrace) File "/usr/local/lib/python3.7/dist-packages/urllib3/packages/six.py", line 770, in reraise raise value File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 710, in urlopen chunked=chunked, File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 451, in _make_request self._raise_timeout(err=e, url=url, timeout_value=read_timeout) File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 341, in _raise_timeout self, url, "Read timed out. (read timeout=%s)" % timeout_value urllib3.exceptions.ReadTimeoutError: AWSHTTPSConnectionPool(host='swf.eu-west-1.amazonaws.com', port=443): Read timed out. (read timeout=60) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/multiprocess/process.py", line 297, in _bootstrap self.run() File "/usr/local/lib/python3.7/dist-packages/multiprocess/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_supervisor.py", line 29, in wrapped return func(*args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_named_mixin.py", line 22, in wrapped return method(self, *args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/poller.py", line 75, in start response = self.poll_with_retry() File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/poller.py", line 169, in poll_with_retry response = poll(task_list, identity=identity) File "/usr/local/lib/python3.7/dist-packages/simpleflow/utils/retry.py", line 70, in decorated return func(*args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_named_mixin.py", line 22, in wrapped return method(self, *args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/decider/base.py", line 137, in poll return simpleflow.swf.mapper.actors.Decider.poll(self, task_list, identity, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/mapper/actors/decider.py", line 93, in poll **kwargs, File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/mapper/core.py", line 339, in poll_for_decision_task **remove_none(kwargs), File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 535, in _api_call return self._make_api_call(operation_name, kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 964, in _make_api_call operation_model, request_dict, request_context File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 986, in _make_request return self._endpoint.make_request(operation_model, request_dict) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 119, in make_request return self._send_request(request_dict, operation_model) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 207, in _send_request exception, File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 361, in _needs_retry request_dict=request_dict, File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 412, in emit return self._emitter.emit(aliased_event_name, **kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 256, in emit return self._emit(event_name, kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 239, in _emit response = handler(**kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 207, in __call__ if self._checker(**checker_kwargs): File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 285, in __call__ attempt_number, response, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 320, in _should_retry return self._checker(attempt_number, response, caught_exception) File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 364, in __call__ attempt_number, response, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 248, in __call__ attempt_number, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 416, in _check_caught_exception raise caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 281, in _do_get_response http_response = self._send(request) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 377, in _send return self.http_session.send(request) File "/usr/local/lib/python3.7/dist-packages/botocore/httpsession.py", line 502, in send raise ReadTimeoutError(endpoint_url=request.url, error=e) botocore.exceptions.ReadTimeoutError: Read timeout on endpoint URL: "https://swf.eu-west-1.amazonaws.com/" 2023-09-26T19:06:05 INFO [process=MainProcess, pid=18087]: starting DeciderPoller(workflow=analysis) on domain XXX ``` Worker: ``` 2023-09-26T19:02:00 INFO [process=MainProcess, pid=17956]: starting ActivityPoller(task_list=analysis) on domain XXX Process Process-57: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 449, in _make_request six.raise_from(e, None) File "", line 3, in raise_from File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 444, in _make_request httplib_response = conn.getresponse() File "/usr/lib/python3.7/http/client.py", line 1348, in getresponse response.begin() File "/usr/lib/python3.7/http/client.py", line 315, in begin version, status, reason = self._read_status() File "/usr/lib/python3.7/http/client.py", line 276, in _read_status line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") File "/usr/lib/python3.7/socket.py", line 589, in readinto return self._sock.recv_into(b) File "/usr/lib/python3.7/ssl.py", line 1071, in recv_into return self.read(nbytes, buffer) File "/usr/lib/python3.7/ssl.py", line 929, in read return self._sslobj.read(len, buffer) socket.timeout: The read operation timed out During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/botocore/httpsession.py", line 474, in send chunked=self._chunked(request.headers), File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 788, in urlopen method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] File "/usr/local/lib/python3.7/dist-packages/urllib3/util/retry.py", line 525, in increment raise six.reraise(type(error), error, _stacktrace) File "/usr/local/lib/python3.7/dist-packages/urllib3/packages/six.py", line 770, in reraise raise value File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 710, in urlopen chunked=chunked, File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 451, in _make_request self._raise_timeout(err=e, url=url, timeout_value=read_timeout) File "/usr/local/lib/python3.7/dist-packages/urllib3/connectionpool.py", line 341, in _raise_timeout self, url, "Read timed out. (read timeout=%s)" % timeout_value urllib3.exceptions.ReadTimeoutError: AWSHTTPSConnectionPool(host='swf.eu-west-1.amazonaws.com', port=443): Read timed out. (read timeout=60) During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/usr/local/lib/python3.7/dist-packages/multiprocess/process.py", line 297, in _bootstrap self.run() File "/usr/local/lib/python3.7/dist-packages/multiprocess/process.py", line 99, in run self._target(*self._args, **self._kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_supervisor.py", line 29, in wrapped return func(*args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_named_mixin.py", line 22, in wrapped return method(self, *args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/poller.py", line 75, in start response = self.poll_with_retry() File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/poller.py", line 169, in poll_with_retry response = poll(task_list, identity=identity) File "/usr/local/lib/python3.7/dist-packages/simpleflow/utils/retry.py", line 70, in decorated return func(*args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/process/_named_mixin.py", line 22, in wrapped return method(self, *args, **kwargs) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/process/worker/base.py", line 79, in poll return simpleflow.swf.mapper.actors.ActivityWorker.poll(self, task_list, identity) File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/mapper/actors/worker.py", line 176, in poll identity=format.identity(identity), File "/usr/local/lib/python3.7/dist-packages/simpleflow/swf/mapper/core.py", line 356, in poll_for_activity_task **remove_none(kwargs), File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 535, in _api_call return self._make_api_call(operation_name, kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 964, in _make_api_call operation_model, request_dict, request_context File "/usr/local/lib/python3.7/dist-packages/botocore/client.py", line 986, in _make_request return self._endpoint.make_request(operation_model, request_dict) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 119, in make_request return self._send_request(request_dict, operation_model) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 207, in _send_request exception, File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 361, in _needs_retry request_dict=request_dict, File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 412, in emit return self._emitter.emit(aliased_event_name, **kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 256, in emit return self._emit(event_name, kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/hooks.py", line 239, in _emit response = handler(**kwargs) File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 207, in __call__ if self._checker(**checker_kwargs): File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 285, in __call__ attempt_number, response, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 320, in _should_retry return self._checker(attempt_number, response, caught_exception) File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 364, in __call__ attempt_number, response, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 248, in __call__ attempt_number, caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/retryhandler.py", line 416, in _check_caught_exception raise caught_exception File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 281, in _do_get_response http_response = self._send(request) File "/usr/local/lib/python3.7/dist-packages/botocore/endpoint.py", line 377, in _send return self.http_session.send(request) File "/usr/local/lib/python3.7/dist-packages/botocore/httpsession.py", line 502, in send raise ReadTimeoutError(endpoint_url=request.url, error=e) botocore.exceptions.ReadTimeoutError: Read timeout on endpoint URL: "https://swf.eu-west-1.amazonaws.com/" 2023-09-26T19:02:55 INFO [process=MainProcess, pid=17986]: starting ActivityPoller(task_list=analysis) on domain XXX ```