Errors when stress testing vulcan

[PierreLesouhaitier]

When using the code below :

import deepomatic
import base64
from multiprocessing import Pool
from tqdm import tqdm
import traceback

APP_ID = ""
API_KEY = ""

VAL_FILE = "/tmp/image_path.txt"

deepomatic_client = deepomatic.Client(APP_ID, API_KEY)

RECOGNITION_VERSION = "10574"
NUMBER_IMAGE_TO_TEST = 1000
NUMBER_PROCESS = 100
TIMEOUT_ON_TASK = 600

def parse_image_digits_file(input_files):
    """
    Parse a file used as input for digits
    """
    return [x.strip().split(' ')[0] for x in open(input_files).readlines()][0:NUMBER_IMAGE_TO_TEST]

def classify_one_image(image_path):
    """
    Classify one image given its path with the client 
    """
    try:
        version = deepomatic_client.RecognitionVersion.retrieve(RECOGNITION_VERSION)
        encoded_image = base64.b64encode(open(image_path, "rb").read())
        task = version.inference(inputs=[deepomatic.ImageInput(encoded_image, encoding="base64")], return_task=True)
        task.wait(timeout=TIMEOUT_ON_TASK)
    except:
        print(traceback.format_exc())

def classify_list_vulcain(image_list):
    """
    Classify a list of path using deepomatic client
    """
    p = Pool(processes=NUMBER_PROCESS)
    for _ in tqdm(p.imap_unordered(classify_one_image, image_list), total=len(image_list), desc='Classify_images'):
        pass
    p.close()
    p.join()

image_list = parse_image_digits_file(VAL_FILE)
classify_list_vulcain(image_list)

I get the following errors (no matter the value of TIMEOUT_ON_TASK):

Traceback (most recent call last):
  File "training_pipeline/minimal_stress_test_vulcain.py", line 33, in classify_one_image
    task = version.inference(inputs=[deepomatic.ImageInput(encoded_image, encoding="base64")], return_task=True)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/utils.py", line 40, in inference
    result = self._helper.post(self._uri(pk=self._pk, suffix='/inference'), content_type=content_type, data=data)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/http_helper.py", line 205, in post
    return self.make_request(requests.post, resource, params, data, content_type, files)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/http_helper.py", line 168, in make_request
    response = func(resource, params=params, data=data, files=files, headers=headers, verify=self.verify, stream=stream)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 112, in post
    return request('post', url, data=data, json=json, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 58, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 618, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 508, in send
    raise ConnectionError(e, request=request)
ConnectionError: HTTPSConnectionPool(host='api.deepomatic.com', port=443): Max retries exceeded with url: /v0.7/recognition/versions/10574/inference (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f0ae8917190>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',))

and

Traceback (most recent call last):
  File "training_pipeline/minimal_stress_test_vulcain.py", line 33, in classify_one_image
    task = version.inference(inputs=[deepomatic.ImageInput(encoded_image, encoding="base64")], return_task=True)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/utils.py", line 43, in inference
    task.wait()
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/resources/task.py", line 47, in wait
    self._wait_result(timeout)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/resources/task.py", line 54, in _wait_result
    while self['status'] == "pending":
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/resource.py", line 71, in __getitem__
    return self.data()[key]
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/resource.py", line 62, in data
    self.refresh()
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/resource.py", line 51, in refresh
    self._data = self._helper.get(self._uri(pk=self._pk))
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/http_helper.py", line 193, in get
    return self.make_request(requests.get, resource, params)
  File "/usr/local/lib/python2.7/dist-packages/deepomatic/http_helper.py", line 168, in make_request
    response = func(resource, params=params, data=data, files=files, headers=headers, verify=self.verify, stream=stream)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 72, in get
    return request('get', url, params=params, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/api.py", line 58, in request
    return session.request(method=method, url=url, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 508, in request
    resp = self.send(prep, **send_kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/sessions.py", line 618, in send
    r = adapter.send(request, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/requests/adapters.py", line 508, in send
    raise ConnectionError(e, request=request)
ConnectionError: HTTPSConnectionPool(host='api.deepomatic.com', port=443): Max retries exceeded with url: /v0.7/tasks/289570962/ (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f0ab31afa50>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',))

[maingoh]

Maybe 100 process is too violent. Apparently it is reproducable with 5 simultaneous requests also. We should also test with a threadpool instead of a process pool.

[PierreLesouhaitier]

• Can be reproduced easily with NUMBER_THREAD=20 (~2000 queries)
• Can take a bit more time for NUMBER_THREAD=10

[maingoh]

Not sure it is still reproductible