pavelvelikhov
commented
8 years ago This program takes 5 seconds to parse
import csv
import sys
from collections import namedtuple
import json
import numpy as np
from dateutil.parser import parse
from pythonql.PQTuple import pq_wrap,pq_flatten
fields = ['', 'event_date', 'event_name', 'client_id', 'lead_id', 'visitor_id', 'partner_id', 'address', 'amount', 'birthdate', 'card_type', 'cardholder_name', 'city', 'credit_limit', 'current_duration', 'domain', 'duration', 'effective_amount', 'email', 'first_name', 'gender', 'interest_rate', 'last_name', 'lead_source', 'middle_name', 'mobile_head', 'mobile_number', 'new_amount', 'new_duration', 'new_interest_rate', 'pan_tail', 'query', 'referer', 'region', 'repaid_amount', 'repaid_interest', 'user_agent', 'utm_campaign', 'webmaster_id', 'reason', 'dt']
schema = {f:i for (i,f) in enumerate(fields)}
out_fields = ['event','date']
out_schema = {f:i for (i,f) in enumerate(out_fields)}
csv.field_size_limit(sys.maxsize)
data = []
f = open('sm_cust_journey.csv')
rd = csv.reader(f)
for (i, (cust_id,cj_data)) in enumerate(rd):
steps = json.loads(cj_data)
data.append( {'id':cust_id, 'cj': steps } )
if i==1000:
break
# compute number of new users, registrations, scorecard checks, accepts, issued, repaid, npl 1, 30, 60, 90
events =( select [
select 'reg' as event, parse(e.event_date) as dt, channel
for e in [item for item in cj_data if item.event_name=="NAME_SAVED"]
count c1
where c1 == 0,
select 'repaid' as event, parse(e2.event_date) as dt, channel
for e in [item for item in cj_data if item.event_name=="LOAN_IS_REPAID"]
count c1
where c1 == 0
for e2 in [item for item in cj_data if item.event_name=="LOAN_ISSUED"]
count c2
where c2 == 0,
select 'np1' as event, parse(e.event_date) as dt, channel
for e in [item for item in cj_data if item.event_name=="LOAN_ISSUED"]
count c1
where c1 == 0
let issue_date = parse(e.event_date)
where not (
select item
for item in cj_data
where item.event_name=="LOAN_IS_REPAID" and (parse(item.event_date) - issue_date ).days < 31 )
]
for cj in data
let cj_data = list( pq_wrap(cj["cj"], schema) )
let channel = try {[item for item in cj_data if item.event_name=="NEW_USER_DETECTED" ][0].reason} except {None}
where not channel is None )
res = (select month, year, channel, event, len(e) as cnt
for e in pq_flatten(events)
let event = e.event
group by e.dt.month as month, e.dt.year as year, e.channel as channel, event
order by year,month)
print(res)
ericvergnaud
SimonStPeter
jimidle
pinaraf
thisiscam
parrt
millergarym
ricardojuanpalmaduran
SimonSntPeter
KvanTTT
sharwell
I have a grammar of Python3 (from the antlr4 grammar repository), extended with query language constructs. The grammar file is here: Grammar file
The whole project is here: PythonQL
This tiny program parses in milliseconds with the Java runtime, but takes about 1.5 seconds in python (after the recent fix, before it was over 2 seconds).
Here is a profiler trace just in case (I left only the relevant entries):
I have attached a file that parses for 7 seconds on my Macbook Pro as well.
I'd be happy to reduce this case to a minimal case for debugging, but don't really know where to start.
The grammar doesn't seem to have any problems like ambiguity, etc.