slipstream / SlipStreamJobEngine

SlipStream distributed job engine
Apache License 2.0
0 stars 0 forks source link

Thread blocked because of ss api #42

Closed 0xbase12 closed 6 years ago

0xbase12 commented 6 years ago

Thanks to new stack trace feature I was able to know where each thread was blocked. After enhancing the job execution and throughput, apparently the ss api is blocking while updating the cimi resources. This occurring because api use request which is not thread safe (supposed at certain level). The fix is to create a new instance for each thread of ss api to avoid such conflict. The patch is actually running on Nuvla.

2018-08-07 14:33:37,180 - ERROR - root - MainThread - thread Thread-1:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  120,
  '_thread_worker',
  'func = queue.get()'),
 ('/usr/lib64/python2.7/Queue.py', 168, 'get', 'self.not_empty.wait()'),
 ('/usr/lib64/python2.7/threading.py', 339, 'wait', 'waiter.acquire()')]
2018-08-07 14:33:37,182 - ERROR - root - MainThread - thread Thread-3590:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  1082,
  'run',
  'self.finished.wait(self.interval)'),
 ('/usr/lib64/python2.7/threading.py',
  622,
  'wait',
  'self.__cond.wait(timeout, balancing)'),
 ('/usr/lib64/python2.7/threading.py', 362, 'wait', '_sleep(delay)')]
2018-08-07 14:33:37,189 - ERROR - root - MainThread - thread job_processor_Wendy_2:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,199 - ERROR - root - MainThread - thread job_processor_Wendy_8:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,203 - ERROR - root - MainThread - thread job_processor_Wendy_7:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  176,
  'update_vm',
  'cimi_vm = self._create_cimi_vm(vm_id, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  262,
  '_create_cimi_vm',
  "orderby='price:unitCost', first=0, last=1)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  644,
  'cimi_search',
  'resp_json = self._cimi_put(resource_type=resource_type, data=cimi_params, params=query_params)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  550,
  '_cimi_put',
  "return self._cimi_request('PUT', uri, params=params, json=json, data=data)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  445,
  'send',
  'timeout=timeout'),
 ('/usr/lib/python2.7/site-packages/urllib3/connectionpool.py',
  600,
  'urlopen',
  'chunked=chunked)'),
 ('/usr/lib/python2.7/site-packages/urllib3/connectionpool.py',
  377,
  '_make_request',
  'httplib_response = conn.getresponse(buffering=True)'),
 ('/usr/lib64/python2.7/httplib.py', 1113, 'getresponse', 'response.begin()'),
 ('/usr/lib64/python2.7/httplib.py',
  444,
  'begin',
  'version, status, reason = self._read_status()'),
 ('/usr/lib64/python2.7/httplib.py',
  400,
  '_read_status',
  'line = self.fp.readline(_MAXLINE + 1)'),
 ('/usr/lib64/python2.7/socket.py',
  476,
  'readline',
  'data = self._sock.recv(self._rbufsize)')]
2018-08-07 14:33:37,207 - ERROR - root - MainThread - thread job_processor_Wendy_10:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,215 - ERROR - root - MainThread - thread job_processor_Wendy_4:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,223 - ERROR - root - MainThread - thread job_processor_Wendy_3:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,228 - ERROR - root - MainThread - thread job_processor_Wendy_9:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,229 - ERROR - root - MainThread - thread Thread-3:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  473,
  'zk_loop',
  'if retry(self._connect_loop, retry) is STOP_CONNECTING:'),
 ('/usr/lib/python2.7/site-packages/kazoo/retry.py',
  124,
  '__call__',
  'return func(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  512,
  '_connect_loop',
  'status = self._connect_attempt(host, port, retry)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  553,
  '_connect_attempt',
  '[], [], timeout)[0]'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  178,
  'select',
  'return self._select(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  192,
  '_select',
  'return select.select(*args, **kwargs)')]
2018-08-07 14:33:37,234 - ERROR - root - MainThread - thread job_processor_Wendy_6:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  216,
  'handle_vm',
  'self.job.add_affected_resource(cimi_vm_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  95,
  'add_affected_resource',
  'self.add_affected_resources([affected_resource])'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  107,
  'add_affected_resources',
  "self._edit_job('affectedResources', [{'href': id} for id in current_affected_resources_ids])"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  132,
  '_edit_job',
  'response = self.ss_api.cimi_edit(self.id, {attribute_name: attribute_value})'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  580,
  'cimi_edit',
  'return models.CimiResponse(self._cimi_put(resource_id=operation_href, json=data))'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  550,
  '_cimi_put',
  "return self._cimi_request('PUT', uri, params=params, json=json, data=data)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,236 - ERROR - root - MainThread - thread MainThread:
[('/opt/slipstream/job/sbin/job_executor.py', 8, '<module>', 'main(Executor)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/base.py',
  110,
  'main',
  'command().execute()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/base.py',
  105,
  'execute',
  'signal.pause()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/util.py',
  40,
  'print_stack',
  'trace = traceback.extract_stack(sys._current_frames()[th.ident])')]
2018-08-07 14:33:37,240 - ERROR - root - MainThread - thread job_processor_Wendy_5:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,241 - ERROR - root - MainThread - thread Thread-2:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  120,
  '_thread_worker',
  'func = queue.get()'),
 ('/usr/lib64/python2.7/Queue.py', 168, 'get', 'self.not_empty.wait()'),
 ('/usr/lib64/python2.7/threading.py', 339, 'wait', 'waiter.acquire()')]
2018-08-07 14:33:37,245 - ERROR - root - MainThread - thread job_processor_Wendy_1:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:33:37,300 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/62f53f57-a9b5-3d8b-a09b-3f036899600e.
2018-08-07 14:33:37,508 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/b54f1c8b-e71b-31a2-b223-873465ffc956.
2018-08-07 14:33:37,709 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/85a32a37-a4b5-3124-8f50-5ad7b9de4f81.
2018-08-07 14:33:37,910 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/7ab74f65-c381-3b9d-83da-851bbf8bf020.
2018-08-07 14:33:38,116 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/98446717-1a95-372c-a755-5fd36a6fc7c2.
2018-08-07 14:33:38,335 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/6e921315-5739-36ff-ba27-0b6bbe313422.
^C
nuvla ~# tail -f /var/log/slipstream/job/executor.log
2018-08-07 14:36:04,273 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - No VMs to collect.
2018-08-07 14:36:04,358 - INFO - Job - job_processor_Wendy_7 - Great, job/741e7419-1f3a-4591-b961-608c2cd1c618 is now in final state; Removed from queue: success True.
2018-08-07 14:36:04,359 - INFO - Executor - job_processor_Wendy_7 - Successfully finished job/741e7419-1f3a-4591-b961-608c2cd1c618.
2018-08-07 14:36:04,383 - INFO - Executor - job_processor_Wendy_7 -  Wendy - Got new job/87491eea-957d-434c-87b7-68c3fe76d5fc.
2018-08-07 14:36:04,426 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Collect virtual machines started for credential/94478041-d760-40ca-9080-b45bd3c318e1.
2018-08-07 14:36:04,691 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Update existing VM: virtual-machine/d7fcd91a-61b5-3e88-be17-fb203818966a.
2018-08-07 14:36:04,839 - INFO - Job - job_processor_Wendy_7 - Great, job/87491eea-957d-434c-87b7-68c3fe76d5fc is now in final state; Removed from queue: success True.
2018-08-07 14:36:04,839 - INFO - Executor - job_processor_Wendy_7 - Successfully finished job/87491eea-957d-434c-87b7-68c3fe76d5fc.
2018-08-07 14:36:04,862 - INFO - Executor - job_processor_Wendy_7 -  Wendy - Got new job/7648760d-3876-46c4-9b06-1d056dc08472.
2018-08-07 14:36:04,906 - INFO - VirtualMachinesCollectJob - job_processor_Wendy_7 - Collect virtual machines started for credential/9f20f70c-9e5f-4b69-954d-5764defde87f.
2018-08-07 14:37:00,070 - ERROR - root - MainThread - thread Thread-1:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  120,
  '_thread_worker',
  'func = queue.get()'),
 ('/usr/lib64/python2.7/Queue.py', 168, 'get', 'self.not_empty.wait()'),
 ('/usr/lib64/python2.7/threading.py', 339, 'wait', 'waiter.acquire()')]
2018-08-07 14:37:00,072 - ERROR - root - MainThread - thread Thread-3628:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  1082,
  'run',
  'self.finished.wait(self.interval)'),
 ('/usr/lib64/python2.7/threading.py',
  622,
  'wait',
  'self.__cond.wait(timeout, balancing)'),
 ('/usr/lib64/python2.7/threading.py', 362, 'wait', '_sleep(delay)')]
2018-08-07 14:37:00,077 - ERROR - root - MainThread - thread job_processor_Wendy_2:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,083 - ERROR - root - MainThread - thread job_processor_Wendy_8:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,089 - ERROR - root - MainThread - thread job_processor_Wendy_7:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  314,
  'collect_virtual_machines',
  'vms = self.connector_instance.list_instances()'),
 ('/usr/lib/python2.7/site-packages/slipstream/util.py',
  119,
  'overrided_func',
  'return func(self, *args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream_opennebula/OpenNebulaClientCloud.py',
  351,
  'list_instances',
  "vms = eTree.fromstring(self._rpc_execute('one.vmpool.info', -3, -1, -1, -1))"),
 ('/usr/lib/python2.7/site-packages/slipstream_opennebula/OpenNebulaClientCloud.py',
  203,
  '_rpc_execute',
  'remote_function(self._create_session_string(), *args)'),
 ('/usr/lib64/python2.7/xmlrpclib.py',
  1233,
  '__call__',
  'return self.__send(self.__name, args)'),
 ('/usr/lib64/python2.7/xmlrpclib.py',
  1591,
  '__request',
  'verbose=self.__verbose'),
 ('/usr/lib64/python2.7/xmlrpclib.py',
  1273,
  'request',
  'return self.single_request(host, handler, request_body, verbose)'),
 ('/usr/lib64/python2.7/xmlrpclib.py',
  1301,
  'single_request',
  'self.send_content(h, request_body)'),
 ('/usr/lib64/python2.7/xmlrpclib.py',
  1448,
  'send_content',
  'connection.endheaders(request_body)'),
 ('/usr/lib64/python2.7/httplib.py',
  1037,
  'endheaders',
  'self._send_output(message_body)'),
 ('/usr/lib64/python2.7/httplib.py', 881, '_send_output', 'self.send(msg)'),
 ('/usr/lib64/python2.7/httplib.py', 843, 'send', 'self.connect()'),
 ('/usr/lib64/python2.7/httplib.py',
  1251,
  'connect',
  'HTTPConnection.connect(self)'),
 ('/usr/lib64/python2.7/httplib.py',
  824,
  'connect',
  'self.timeout, self.source_address)'),
 ('/usr/lib64/python2.7/socket.py',
  562,
  'create_connection',
  'sock.connect(sa)'),
 ('/usr/lib64/python2.7/socket.py',
  224,
  'meth',
  'return getattr(self._sock,name)(*args)')]
2018-08-07 14:37:00,093 - ERROR - root - MainThread - thread job_processor_Wendy_10:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,097 - ERROR - root - MainThread - thread job_processor_Wendy_4:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,102 - ERROR - root - MainThread - thread job_processor_Wendy_3:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,106 - ERROR - root - MainThread - thread job_processor_Wendy_9:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,108 - ERROR - root - MainThread - thread Thread-3:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  473,
  'zk_loop',
  'if retry(self._connect_loop, retry) is STOP_CONNECTING:'),
 ('/usr/lib/python2.7/site-packages/kazoo/retry.py',
  124,
  '__call__',
  'return func(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  512,
  '_connect_loop',
  'status = self._connect_attempt(host, port, retry)'),
 ('/usr/lib/python2.7/site-packages/kazoo/protocol/connection.py',
  553,
  '_connect_attempt',
  '[], [], timeout)[0]'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  178,
  'select',
  'return self._select(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  192,
  '_select',
  'return select.select(*args, **kwargs)')]
2018-08-07 14:37:00,116 - ERROR - root - MainThread - thread job_processor_Wendy_6:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  216,
  'handle_vm',
  'self.job.add_affected_resource(cimi_vm_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  95,
  'add_affected_resource',
  'self.add_affected_resources([affected_resource])'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  107,
  'add_affected_resources',
  "self._edit_job('affectedResources', [{'href': id} for id in current_affected_resources_ids])"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  132,
  '_edit_job',
  'response = self.ss_api.cimi_edit(self.id, {attribute_name: attribute_value})'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  580,
  'cimi_edit',
  'return models.CimiResponse(self._cimi_put(resource_id=operation_href, json=data))'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  550,
  '_cimi_put',
  "return self._cimi_request('PUT', uri, params=params, json=json, data=data)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,118 - ERROR - root - MainThread - thread MainThread:
[('/opt/slipstream/job/sbin/job_executor.py', 8, '<module>', 'main(Executor)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/base.py',
  110,
  'main',
  'command().execute()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/base.py',
  105,
  'execute',
  'signal.pause()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/util.py',
  40,
  'print_stack',
  'trace = traceback.extract_stack(sys._current_frames()[th.ident])')]
2018-08-07 14:37:00,124 - ERROR - root - MainThread - thread job_processor_Wendy_5:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  43,
  '_process_jobs',
  'return_code = self.job_processor(job)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  75,
  'job_processor',
  'return action_instance.do_work()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  330,
  'do_work',
  'self.collect_virtual_machines()'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  319,
  'collect_virtual_machines',
  'map(self.handle_vm, vms)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  214,
  'handle_vm',
  'cimi_vm_id = self.update_vm(vm_id, exiting_vms, vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/actions/virtual_machines_collect.py',
  195,
  'update_vm',
  'self.ss_api.cimi_edit(cimi_vm_id, cimi_vm)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]
2018-08-07 14:37:00,125 - ERROR - root - MainThread - thread Thread-2:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/kazoo/handlers/threading.py',
  120,
  '_thread_worker',
  'func = queue.get()'),
 ('/usr/lib64/python2.7/Queue.py', 168, 'get', 'self.not_empty.wait()'),
 ('/usr/lib64/python2.7/threading.py', 339, 'wait', 'waiter.acquire()')]
2018-08-07 14:37:00,130 - ERROR - root - MainThread - thread job_processor_Wendy_1:
[('/usr/lib64/python2.7/threading.py',
  785,
  '__bootstrap',
  'self.__bootstrap_inner()'),
 ('/usr/lib64/python2.7/threading.py', 812, '__bootstrap_inner', 'self.run()'),
 ('/usr/lib64/python2.7/threading.py',
  765,
  'run',
  'self.__target(*self.__args, **self.__kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/executor.py',
  53,
  '_process_jobs',
  "job.update_job(state='FAILED', status_message=status_message)"),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  122,
  'update_job',
  'self._edit_job_multi(attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/job/job.py',
  143,
  '_edit_job_multi',
  'response = self.ss_api.cimi_edit(self.id, attributes)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  578,
  'cimi_edit',
  'resource = self.cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  563,
  'cimi_get',
  'resp_json = self._cimi_get(resource_id=resource_id)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  542,
  '_cimi_get',
  "return self._cimi_request('GET', uri, params=params)"),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  521,
  '_cimi_request',
  'data=data)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  221,
  'request',
  'response = self._request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/slipstream/api/api.py',
  218,
  '_request',
  'return super(SessionStore, self).request(*args, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  512,
  'request',
  'resp = self.send(prep, **send_kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/sessions.py',
  622,
  'send',
  'r = adapter.send(request, **kwargs)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  410,
  'send',
  'conn = self.get_connection(request.url, proxies)'),
 ('/usr/lib/python2.7/site-packages/requests/adapters.py',
  314,
  'get_connection',
  'conn = self.poolmanager.connection_from_url(url)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  279,
  'connection_from_url',
  'pool_kwargs=pool_kwargs)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  227,
  'connection_from_host',
  'return self.connection_from_context(request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  240,
  'connection_from_context',
  'return self.connection_from_pool_key(pool_key, request_context=request_context)'),
 ('/usr/lib/python2.7/site-packages/urllib3/poolmanager.py',
  250,
  'connection_from_pool_key',
  'with self.pools.lock:'),
 ('/usr/lib64/python2.7/threading.py',
  173,
  'acquire',
  'rc = self.__block.acquire(blocking)')]