During ETL search stage there is a read timeout during streaming_bulk calls. Might be easily remediated by simply increasing request_timeout arg in function call.
INFO:root:Populating ES with DataPaths...
WARNING:elasticsearch:POST http://search:9200/datapath/doc/_bulk [status:N/A request:10.002s]
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.7/http/client.py", line 1336, in getresponse
response.begin()
File "/usr/local/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 238, in perform_request
method, url, body, retries=Retry(False), headers=request_headers, **kw
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/util/retry.py", line 376, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 423, in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='search', port=9200): Read timed out. (read timeout=10)
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 421, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 416, in _make_request
httplib_response = conn.getresponse()
File "/usr/local/lib/python3.7/http/client.py", line 1336, in getresponse
response.begin()
File "/usr/local/lib/python3.7/http/client.py", line 306, in begin
version, status, reason = self._read_status()
File "/usr/local/lib/python3.7/http/client.py", line 267, in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
File "/usr/local/lib/python3.7/socket.py", line 589, in readinto
return self._sock.recv_into(b)
socket.timeout: timed out
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 238, in perform_request
method, url, body, retries=Retry(False), headers=request_headers, **kw
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 720, in urlopen
method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/util/retry.py", line 376, in increment
raise six.reraise(type(error), error, _stacktrace)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/packages/six.py", line 735, in reraise
raise value
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 672, in urlopen
chunked=chunked,
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 423, in _make_request
web:
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/urllib3/connectionpool.py", line 331, in _raise_timeout
self, url, "Read timed out. (read timeout=%s)" % timeout_value
urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='search', port=9200): Read timed out. (read timeout=10)
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "main.py", line 118, in <module>
main()
File "main.py", line 103, in main
populate_search(db, config['search']['searchURL'])
File "/data/search.py", line 32, in populate_search
populate_search_db(es, query_iterable, index, doc_type)
File "/data/search.py", line 185, in populate_search_db
doc_type=doc_type
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 230, in streaming_bulk
**kwargs
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 116, in _process_bulk_chunk
raise e
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/helpers/actions.py", line 112, in _process_bulk_chunk
resp = client.bulk("\n".join(bulk_actions) + "\n", *args, **kwargs)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/client/utils.py", line 84, in _wrapped
return func(*args, params=params, **kwargs)
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/client/__init__.py", line 1483, in bulk
headers={"content-type": "application/x-ndjson"},
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/transport.py", line 350, in perform_request
timeout=timeout,
File "/root/.local/share/virtualenvs/data-I7nS9QO2/lib/python3.7/site-packages/elasticsearch/connection/http_urllib3.py", line 249, in perform_request
raise ConnectionTimeout("TIMEOUT", str(e), e)
elasticsearch.exceptions.ConnectionTimeout: ConnectionTimeout caused by - ReadTimeoutError(HTTPConnectionPool(host='search', port=9200): Read timed out. (read timeout=10))
During ETL search stage there is a read timeout during
streaming_bulkcalls. Might be easily remediated by simply increasingrequest_timeoutarg in function call.