LaurentDumont
commented
8 years ago Or actually try to properly learn FutureRequests :
def getData(requestsConnection,typeid,regionid,buysell):
url="https://public-crest.eveonline.com/market/{}/orders/{}/?type=https://public-crest.eveonline.com/types/{}/".format(regionid,buysell,typeid)
future=requestsConnection.get(url)
future.typeid=typeid
future.regionid=regionid
future.buysell=buysell
return future for typeid in tqdm(baseitemids,desc="downloading",leave=True):
futures.append(getData(session,typeid,buyregion,'buy'))
futures.append(getData(session,typeid,sellregion,'sell'))
for result in tqdm(as_completed(futures),desc='Processing',total=len(baseitemids)*2,leave=True):
trans = connection.begin()
status=processData(result,connection,orderTable)
if not status:
retryfutures.append(getData(session,result.typeid,result.regionid,result.buysell))
logging.warn("adding {} to retry for {} in {}".format(result.typeid,result.buysell,result.regionid))
trans.commit()
for result in tqdm(as_completed(retryfutures),desc='Processing Failures',total=len(retryfutures),leave=True):
trans = connection.begin()
status=processData(result,connection,orderTable)
if not status:
logging.warn("adding {} to retry for {} in {}".format(result.typeid,result.buysell,result.regionid))
trans.commit()
from sqlalchemy import create_engine, Column, MetaData, Table, Index
from sqlalchemy import Integer, String, Text, Float, Boolean, BigInteger, Numeric, SmallInteger
import time
import requests
from requests_futures.sessions import FuturesSession
import requests_futures
from concurrent.futures import as_completed
import datetime
import csv
import time
from tqdm import tqdm
import sys
import glob
import pprint
from math import floor,log10,fabs
import logging
logging.basicConfig(filename='c:/app/getbuy.log',level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')
def RateLimited(maxPerSecond):
minInterval = 1.0 / float(maxPerSecond)
def decorate(func):
lastTimeCalled = [0.0]
def rateLimitedFunction(*args,**kargs):
elapsed = time.clock() - lastTimeCalled[0]
leftToWait = minInterval - elapsed
if leftToWait>0:
time.sleep(leftToWait)
ret = func(*args,**kargs)
lastTimeCalled[0] = time.clock()
return ret
return rateLimitedFunction
return decorate
def processData(result,connection,orderTable):
try:
resp=result.result()
if resp.status_code==200:
orders=resp.json()
for order in orders['items']:
connection.execute(orderTable.insert(),
orderID=order['id'],
typeID=order['type']['id'],
buy=order['buy'],
volume=order['volume'],
minVolume=order['minVolume'],
price=order['price'],
roundedPrice=round(order['price'],2-int(floor(log10(fabs(order['price']))))),
location=order['location']['id'],
locationName=order['location']['name']
)
return True;
else:
logging.warn("None 200 status. {} Returned: {}".format(resp.typeid,resp.status_code))
return False
except requests.exceptions.ConnectionError as e:
logging.warn(e)
return False;
#except:
# logging.warn("Some other error occured")
# return False;
@RateLimited(150)
def getData(requestsConnection,typeid,regionid,buysell):
url="https://public-crest.eveonline.com/market/{}/orders/{}/?type=https://public-crest.eveonline.com/types/{}/".format(regionid,buysell,typeid)
future=requestsConnection.get(url)
future.typeid=typeid
future.regionid=regionid
future.buysell=buysell
return future
if __name__ == "__main__":
engine = create_engine('sqlite+pysqlite:///c:/app/market.db', echo=False)
metadata = MetaData()
connection = engine.connect()
# build the basic list
baseitemids=[]
files=glob.glob('*.csv')
i=0
print "Pick a file"
for file in files:
print "{}: {}".format(i,file)
i+=1
file=raw_input("File Number:")
with open(files[int(file)], 'rb') as csvfile:
typereader = csv.reader(csvfile, delimiter=',', quotechar='"')
for row in typereader:
baseitemids.append(row[0])
reqs_num_workers = 10
session = FuturesSession(max_workers=reqs_num_workers)
session.headers.update({'UserAgent':'Fuzzwork Market Monitor'});
orderTable = Table('orders',metadata,
Column('orderID',BigInteger,primary_key=True, autoincrement=False),
Column('typeID',Integer),
Column('buy',Integer),
Column('volume',BigInteger),
Column('minVolume',BigInteger),
Column('price',Numeric(scale=4,precision=19)),
Column('roundedPrice',Numeric(scale=4,precision=19)),
Column('location',Integer),
Column('locationName',String)
)
Index("orders_1",orderTable.c.typeID)
Index("orders_2",orderTable.c.typeID,orderTable.c.buy)
Index("orders_4",orderTable.c.typeID,orderTable.c.buy,orderTable.c.location)
Index("orders_3",orderTable.c.typeID,orderTable.c.buy,orderTable.c.price)
Index("orders_5",orderTable.c.typeID,orderTable.c.buy,orderTable.c.roundedPrice)
metadata.create_all(engine,checkfirst=True)
connection.execute(orderTable.delete())
print 'Route'
print '1: Jita to Dodixie'
print '2: Dodixie to Jita'
print '3: Amarr to Jita'
print '4: Jita to Amarr'
print '5: Amarr to Dodixie'
print '6: Dodixie to Amarr'
file=raw_input("Route Number:")
if (int(file)==1):
buyregion=10000032
sellregion=10000002
elif (int(file)==2):
buyregion=10000002
sellregion=10000032
elif (int(file)==3):
buyregion=10000002
sellregion=10000043
elif (int(file)==4):
buyregion=10000043
sellregion=10000002
elif (int(file)==5):
buyregion=10000032
sellregion=10000043
elif (int(file)==6):
buyregion=10000043
sellregion=10000032
else:
buyregion=10000032
sellregion=10000002
futures=[]
retryfutures=[]
for typeid in tqdm(baseitemids,desc="downloading",leave=True):
futures.append(getData(session,typeid,buyregion,'buy'))
futures.append(getData(session,typeid,sellregion,'sell'))
for result in tqdm(as_completed(futures),desc='Processing',total=len(baseitemids)*2,leave=True):
trans = connection.begin()
status=processData(result,connection,orderTable)
if not status:
retryfutures.append(getData(session,result.typeid,result.regionid,result.buysell))
logging.warn("adding {} to retry for {} in {}".format(result.typeid,result.buysell,result.regionid))
trans.commit()
for result in tqdm(as_completed(retryfutures),desc='Processing Failures',total=len(retryfutures),leave=True):
trans = connection.begin()
status=processData(result,connection,orderTable)
if not status:
logging.warn("adding {} to retry for {} in {}".format(result.typeid,result.buysell,result.regionid))
trans.commit()
https://twistedmatrix.com/trac/wiki