espeed
commented
11 years ago Bulbs does not support multithreading at the moment, but it would not take too much to get it there.
However, in this case (an in most cases), a Gremlin script that encapsulates all the lookups/search into a single request will be more performant than issuing multiple requests in multiple threads because you won't have the HTTP overhead.
See these examples:
not sure whether bulbs support multithread but it blow up when I try to run below code, would you mind having a look and see whether this is an issue with bulbs or I am doing the threading wrongly (it run ok if parallel set to False)? thanks
import collections import time from bulbs.titan import Graph, Config import thread
def slice( names, size ): m = collections.defaultdict( list ) for i, name in enumerate( names ): m[ i/size ].append( name ) return m.values()
def search( names, graph ): return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ]
def titan(): config = Config( 'http://localhost:8182/graphs/graph' ) parallel = True graph = Graph( config ) for x in slice( map( lambda x:'name{0}'.format( x ), xrange( 100 ) ), 30 ): if parallel: thread.start_new_thread( search, (x, graph) ) else: search( x, graph )
titan()
while 1: pass
Unhandled exception in thread started by <function search at 0x10b95c050> Traceback (most recent call last): File "test.py", line 21, in search return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ] File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup Unhandled exception in thread started by <function search at 0x10b95c050> resp = self.client.lookup_vertex(self.index_name,key,value) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex return self.request.get(path,params)
Traceback (most recent call last): File "test.py", line 21, in search return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ] File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup resp = self.client.lookup_vertex(self.index_name,key,value) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex return self.request.get(path,params) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get Unhandled exception in thread started by <function search at 0x10b95c050> Traceback (most recent call last): return self.request(GET, path, params) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request http_resp = self.http.request(uri, method, body, headers) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get return self.request(GET, path, params) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request File "test.py", line 21, in search return [ list( graph.vertices.index.lookup( username=name ) ) for name in names ](response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/index.py", line 270, in lookup http_resp = self.http.request(uri, method, body, headers) resp = self.client.lookup_vertex(self.index_name,key,value) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/titan/client.py", line 337, in lookup_vertex File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request (response, content) = self._conn_request(conn, request_uri, method, body, headers) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1286, in _conn_request response = conn.getresponse() File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 1027, in getresponse return self.request.get(path,params) response.begin() File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 407, in begin version, status, reason = self._read_status() File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 98, in get return self.request(GET, path, params) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/bulbs-0.3.14_20130803-py2.7.egg/bulbs/rest.py", line 181, in request http_resp = self.http.request(uri, method, body, headers) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1570, in request (response, content) = self._request(conn, authority, uri, request_uri, method, body, headers, redirections, cachekey) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1317, in _request (response, content) = self._conn_request(conn, request_uri, method, body, headers) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1290, in _conn_request conn.connect() File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 913, in connect raise socket.error, msg socket File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/httplib.py", line 365, in _read_status (response, content) = self._conn_request(conn, request_uri, method, body, headers) File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 1290, in _conn_request conn.connect() File "/Users/xyz/.env/zyx/lib/python2.7/site-packages/httplib2/init.py", line 913, in connect raise socket.error, msg socket.error: . line = self.fp.readline() getaddrinfo returns an empty listerror File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/socket.py", line 430, in readline : data = recv(1) socket.error: [Errno 57] Socket is not connectedgetaddrinfo returns an empty list