talolard / MarketVectors

Implementations for my blog post [here](https://medium.com/@TalPerry/deep-learning-the-stock-market-df853d139e02#.flflpo3xf)
MIT License
260 stars 169 forks source link

pivot_table cause precision loss #2

Closed chutaklee closed 7 years ago

chutaklee commented 7 years ago

https://github.com/pandas-dev/pandas/issues/15091 I fixed it by passing a numpy array. Ugly, but it works.

c_2_o = pd.DataFrame()
h_2_o = pd.DataFrame()
l_2_o = pd.DataFrame()
c_2_h = pd.DataFrame()
h_2_l = pd.DataFrame()
c1_c0 = pd.DataFrame()
vol = pd.DataFrame()

def make_inputs(filepath):
    D = pd.read_csv(filepath,header=None,names=['ticker','o','h','l','c','v']) 
    #Load the dataframe with headers
    D.index = pd.to_datetime(D.index,format='%Y%m%d') 
    # Set the indix to a datetime
    ticker = str(get_ticker(filepath))

    c_2_o[(ticker + '_c_2_o')] = zscore(ret(D.o,D.c))
    h_2_o[(ticker + '_h_2_o')] = zscore(ret(D.o,D.h))
    l_2_o[(ticker + '_l_2_o')] = zscore(ret(D.o,D.l))
    c_2_h[(ticker + '_c_2_h')] = zscore(ret(D.h,D.c))
    h_2_l[(ticker + '_h_2_l')] = zscore(ret(D.h,D.l))
    c1_c0[(ticker + '_c1_c0')] = ret(D.c,D.c.shift(-1)).fillna(0) #Tommorows return 
    vol[(ticker + '_vol')] = zscore(D.v)

for f in os.listdir(datapath):
    filepath = os.path.join(datapath,f)
    if filepath.endswith('.csv'):
        make_inputs(filepath)

dates = c_2_o.index
pivot = list(c_2_o.columns) +list( h_2_o.columns) +list( l_2_o.columns) + list(c_2_h.columns) + list(h_2_l.columns) + list(c1_c0.columns) + list(vol.columns)
values = np.concatenate((c_2_o.values, h_2_o.values, l_2_o.values, c_2_h.values, h_2_l.values, c1_c0.values, vol.values), axis=1)
flat = pd.DataFrame(values, index=dates, columns=pivot)