set_history(), history() not working on minute level

[rhodan]

again,

I wonder if history() function works on minute level data.

It works well on daily basis.

from zipline.algorithm import TradingAlgorithm
from zipline.sources.simulated import RandomWalkSource
from zipline.utils import tradingcalendar as calendar_nyse
from datetime import datetime
import pytz
from zipline.api import *

start = datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc)
end = datetime(2014, 1, 10, 0, 0, 0, 0, pytz.utc)

source = RandomWalkSource(start_prices={0: 100, 1: 500}, freq='minute', start=start,
                 end=end,calendar=calendar_nyse)

def initialize(context):
    add_history(5, '1m', 'price')

def handle_data(context, source):
    recent_prices = history(5, '1m', 'price').mean()
    print("recent", recent_prices)

algo = TradingAlgorithm(initialize=initialize, handle_data=handle_data,
                        data_frequency='minute', identifiers=source.sids)

perf = algo.run(source)

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-14-843075e403bf> in <module>()
     29                         data_frequency='minute', identifiers=source.sids)
     30 
---> 31 perf = algo.run(source)

C:\Anaconda3\lib\site-packages\zipline\algorithm.py in run(self, source, overwrite_sim_params, benchmark_return_source)
    599         # perf dictionary
    600         perfs = []
--> 601         for perf in self.gen:
    602             perfs.append(perf)
    603 

C:\Anaconda3\lib\site-packages\zipline\gens\tradesimulation.py in transform(self, stream_in)
    119                         date,
    120                         snapshot,
--> 121                         self.algo.instant_fill,
    122                     )
    123                     # Perf messages are only emitted if the snapshot contained

C:\Anaconda3\lib\site-packages\zipline\gens\tradesimulation.py in _process_snapshot(self, dt, snapshot, instant_fill)
    296 
    297         if any_trade_occurred:
--> 298             new_orders = self._call_handle_data()
    299             for order in new_orders:
    300                 perf_process_order(order)

C:\Anaconda3\lib\site-packages\zipline\gens\tradesimulation.py in _call_handle_data(self)
    325             self.algo,
    326             self.current_data,
--> 327             self.simulation_dt,
    328         )
    329         orders = self.algo.blotter.new_orders

C:\Anaconda3\lib\site-packages\zipline\utils\events.py in handle_data(self, context, data, dt)
    190     def handle_data(self, context, data, dt):
    191         for event in self._events:
--> 192             event.handle_data(context, data, dt, context.trading_environment)
    193 
    194 

C:\Anaconda3\lib\site-packages\zipline\utils\events.py in handle_data(self, context, data, dt, env)
    208         """
    209         if self.rule.should_trigger(dt, env):
--> 210             self.callback(context, data)
    211 
    212 

C:\Anaconda3\lib\site-packages\zipline\algorithm.py in handle_data(self, data)
    372             self.history_container.update(data, self.datetime)
    373 
--> 374         self._handle_data(self, data)
    375 
    376         # Unlike trading controls which remain constant unless placing an

<ipython-input-14-843075e403bf> in handle_data(context, source)
     21     #order(sid(0), 1)
     22     #record(rndm=source[sid(0)].price)
---> 23     recent_prices = history(5, '1m', 'price').mean()
     24     print("recent", recent_prices)
     25     pass

C:\Anaconda3\lib\site-packages\zipline\utils\api_support.py in wrapped(*args, **kwargs)
     49     def wrapped(*args, **kwargs):
     50         # Get the instance and call the method
---> 51         return getattr(get_algo_instance(), f.__name__)(*args, **kwargs)
     52     # Add functor to zipline.api
     53     setattr(zipline.api, f.__name__, wrapped)

C:\Anaconda3\lib\site-packages\zipline\utils\api_support.py in wrapped_method(self, *args, **kwargs)
     96             if not self.initialized:
     97                 raise exception
---> 98             return method(self, *args, **kwargs)
     99         return wrapped_method
    100     return decorator

C:\Anaconda3\lib\site-packages\zipline\algorithm.py in history(self, bar_count, frequency, field, ffill)
   1273             ffill,
   1274         )
-> 1275         return self.history_container.get_history(history_spec, self.datetime)
   1276 
   1277     ####################

C:\Anaconda3\lib\site-packages\zipline\history\history_container.py in get_history(self, history_spec, algo_dt)
    902                 digest_frame,
    903                 self.last_known_prior_values,
--> 904                 raw=True
    905             )
    906         last_period = self.frame_to_series(field, buffer_frame, self.sids)

C:\Anaconda3\lib\site-packages\zipline\history\history_container.py in ffill_buffer_from_prior_values(freq, field, buffer_frame, digest_frame, pv_frame, raw)
     54         buffer_values = buffer_frame.values
     55 
---> 56     nan_sids = pd.isnull(buffer_values[0])
     57     if np.any(nan_sids) and len(digest_values):
     58         # If we have any leading nans in the buffer and we have a non-empty

IndexError: index 0 is out of bounds for axis 0 with size 0

[ssanderson]

@llllllllll I think you recently merged a change that fixed a bug that was crashing with the same error Does this ring a bell to you?

[llllllllll]

The recent indexerror I dealt with was in the blaze loader, I think this my be a dup of: #447

[rhodan]

I found it works well with zipline 0.7.0

from zipline.algorithm import TradingAlgorithm
from zipline.api import symbol, order, record
from zipline.sources.simulated import RandomWalkSource
from zipline.utils import tradingcalendar as calendar_nyse
from datetime import datetime
import pytz
from zipline.api import *

start_date = datetime(2014, 1, 1, 0, 0, 0, 0, pytz.utc)
end_date = datetime(2014, 2, 10, 0, 0, 0, 0, pytz.utc)

source = RandomWalkSource(start_prices={0: 100, 1: 500}, freq='minute', start=start_date,
                 end=end_date,calendar=calendar_nyse)

def initialize(context):
    add_history(5, '1m', 'price')

def handle_data(context, source):

    price_history = history(5, '1m', 'price')
    print("recent", price_history)

algo = TradingAlgorithm(initialize=initialize, handle_data=handle_data)

perf = algo.run(source)

recent                                    0           1
2013-12-31 20:57:00+00:00        NaN         NaN
2013-12-31 20:58:00+00:00        NaN         NaN
2013-12-31 20:59:00+00:00        NaN         NaN
2013-12-31 21:00:00+00:00        NaN         NaN
2014-01-02 14:31:00+00:00  99.916265  500.154655
recent                                     0           1
2013-12-31 20:58:00+00:00         NaN         NaN
2013-12-31 20:59:00+00:00         NaN         NaN
2013-12-31 21:00:00+00:00         NaN         NaN
2014-01-02 14:31:00+00:00   99.916265  500.154655
......

then I tried resample minute level data to daily frequency

source = RandomWalkSource(start_prices={0: 100, 1: 500}, freq='minute', start=start_date,
                 end=end_date,calendar=calendar_nyse)

def initialize(context):
    add_history(5, '1d', 'price')

def handle_data(context, source):

    price_history = history(5, '1d', 'price')
    print("recent", price_history)

and I get indexerror

......
-> 1354             val = getitem(key)
   1355             return Timestamp(val, offset=self.offset, tz=self.tz)
   1356         else:

IndexError: index 0 is out of bounds for axis 0 with size 0

I think its quiet important for traders to use daily prices on minute level.

[freddiev4]

Hi @rhodan I'm going to close this issue in favor of other issues opened right now that also use our most current zipline API. Feel free to open a new issue or comment on one of the existing issues labeled with Minute Mode 😄