ValueError: Shape of passed values is (1, 266), indices imply (1, 265)

[CptanPanic]

I just installed pyfolio and am trying a quick test, but get an error. I am on ubuntu 16.04 and python3.

<bound method NDFrame.head of                              return
index                              
2017-01-01 00:00:00+00:00  0.000000
2017-01-02 00:00:00+00:00  0.000000
2017-01-03 00:00:00+00:00  0.000000
2017-01-04 00:00:00+00:00  0.000000
2017-01-05 00:00:00+00:00  0.000000
2017-01-06 00:00:00+00:00  0.000000
2017-01-07 00:00:00+00:00  0.000000
2017-01-08 00:00:00+00:00  0.000000
2017-01-09 00:00:00+00:00  0.000000
2017-01-10 00:00:00+00:00  0.000000
2017-01-11 00:00:00+00:00  0.000000
2017-01-12 00:00:00+00:00  0.000000
2017-01-13 00:00:00+00:00  0.000000
2017-01-14 00:00:00+00:00  0.000000
2017-01-15 00:00:00+00:00  0.000000
2017-01-16 00:00:00+00:00  0.000000
2017-01-17 00:00:00+00:00  0.000000
2017-01-18 00:00:00+00:00  0.000000
2017-01-19 00:00:00+00:00  0.000000
2017-01-20 00:00:00+00:00  0.000000
2017-01-21 00:00:00+00:00  0.000000
2017-01-22 00:00:00+00:00  0.000000
2017-01-23 00:00:00+00:00  0.000000
2017-01-24 00:00:00+00:00  0.000000
2017-01-25 00:00:00+00:00  0.000000
2017-01-26 00:00:00+00:00  0.000000
2017-01-27 00:00:00+00:00 -0.001159
2017-01-28 00:00:00+00:00  0.001204
2017-01-29 00:00:00+00:00 -0.002386
2017-01-30 00:00:00+00:00 -0.002529
...                             ...
2017-08-24 00:00:00+00:00  0.000178
2017-08-25 00:00:00+00:00  0.006851
2017-08-26 00:00:00+00:00  0.004342
2017-08-27 00:00:00+00:00  0.016193
2017-08-28 00:00:00+00:00 -0.009291
2017-08-29 00:00:00+00:00  0.050239
2017-08-30 00:00:00+00:00  0.028503
2017-08-31 00:00:00+00:00 -0.008198
2017-09-01 00:00:00+00:00 -0.002621
2017-09-02 00:00:00+00:00 -0.080064
2017-09-03 00:00:00+00:00 -0.013076
2017-09-04 00:00:00+00:00  0.000000
2017-09-05 00:00:00+00:00  0.000000
2017-09-06 00:00:00+00:00  0.004541
2017-09-07 00:00:00+00:00  0.002544
2017-09-08 00:00:00+00:00 -0.087556
2017-09-09 00:00:00+00:00  0.000000
2017-09-10 00:00:00+00:00  0.000000
2017-09-11 00:00:00+00:00  0.000000
2017-09-12 00:00:00+00:00 -0.042191
2017-09-13 00:00:00+00:00 -0.002261
2017-09-14 00:00:00+00:00 -0.018250
2017-09-15 00:00:00+00:00  0.000000
2017-09-16 00:00:00+00:00  0.000000
2017-09-17 00:00:00+00:00 -0.013981
2017-09-18 00:00:00+00:00  0.115473
2017-09-19 00:00:00+00:00 -0.032337
2017-09-20 00:00:00+00:00  0.001545
2017-09-21 00:00:00+00:00 -0.068097
2017-09-22 00:00:00+00:00  0.000000

[265 rows x 1 columns]>

Entire data start date: 2017-01-01
Entire data end date: 2017-09-22

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in create_block_manager_from_blocks(blocks, axes)
   4295 
-> 4296         mgr = BlockManager(blocks, axes)
   4297         mgr._consolidate_inplace()

/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in __init__(self, blocks, axes, do_integrity_check, fastpath)
   2794         if do_integrity_check:
-> 2795             self._verify_integrity()
   2796 

/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in _verify_integrity(self)
   3005             if block._verify_integrity and block.shape[1:] != mgr_shape[1:]:
-> 3006                 construction_error(tot_items, block.shape[1:], self.axes)
   3007         if len(self.items) != tot_items:

/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in construction_error(tot_items, block_shape, axes, e)
   4279     raise ValueError("Shape of passed values is {0}, indices imply {1}".format(
-> 4280         passed, implied))
   4281 

ValueError: Shape of passed values is (1, 266), indices imply (1, 265)

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-29-14ea3ae20a60> in <module>()
----> 1 pf.create_full_tear_sheet(returns)

/usr/local/lib/python3.5/dist-packages/pyfolio/tears.py in create_full_tear_sheet(returns, positions, transactions, market_data, benchmark_rets, slippage, live_start_date, sector_mappings, bayesian, round_trips, estimate_intraday, hide_positions, cone_std, bootstrap, unadjusted_returns, risk, style_factor_panel, sectors, caps, shares_held, volumes, percentile, set_context)
    179         benchmark_rets=benchmark_rets,
    180         bootstrap=bootstrap,
--> 181         set_context=set_context)
    182 
    183     create_interesting_times_tear_sheet(returns,

/usr/local/lib/python3.5/dist-packages/pyfolio/plotting.py in call_w_context(*args, **kwargs)
     50         if set_context:
     51             with plotting_context(), axes_style():
---> 52                 return func(*args, **kwargs)
     53         else:
     54             return func(*args, **kwargs)

/usr/local/lib/python3.5/dist-packages/pyfolio/tears.py in create_returns_tear_sheet(returns, positions, transactions, live_start_date, cone_std, benchmark_rets, bootstrap, return_fig)
    453                              transactions=transactions,
    454                              bootstrap=bootstrap,
--> 455                              live_start_date=live_start_date)
    456 
    457     plotting.show_worst_drawdown_periods(returns)

/usr/local/lib/python3.5/dist-packages/pyfolio/plotting.py in show_perf_stats(returns, factor_returns, positions, transactions, live_start_date, bootstrap)
    642         factor_returns=factor_returns,
    643         positions=positions,
--> 644         transactions=transactions)
    645 
    646     if live_start_date is not None:

/usr/local/lib/python3.5/dist-packages/pyfolio/timeseries.py in perf_stats(returns, factor_returns, positions, transactions)
    718     stats = pd.Series()
    719     for stat_func in SIMPLE_STAT_FUNCS:
--> 720         stats[STAT_FUNC_NAMES[stat_func.__name__]] = stat_func(returns)
    721 
    722     if positions is not None:

/usr/local/lib/python3.5/dist-packages/empyrical/stats.py in calmar_ratio(returns, period, annualization)
    349     """
    350 
--> 351     max_dd = max_drawdown(returns=returns)
    352     if max_dd < 0:
    353         temp = annual_return(

/usr/local/lib/python3.5/dist-packages/empyrical/stats.py in max_drawdown(returns)
    227         returns = returns.values
    228 
--> 229     cumulative = np.insert(cum_returns(returns, starting_value=100), 0, 100)
    230     max_return = np.fmax.accumulate(cumulative)
    231 

~/.local/lib/python3.5/site-packages/numpy/lib/function_base.py in insert(arr, obj, values, axis)
   5063         new[slobj] = arr[slobj2]
   5064         if wrap:
-> 5065             return wrap(new)
   5066         return new
   5067     elif indices.size == 0 and not isinstance(obj, np.ndarray):

/usr/local/lib/python3.5/dist-packages/pandas/core/generic.py in __array_wrap__(self, result, context)
    987     def __array_wrap__(self, result, context=None):
    988         d = self._construct_axes_dict(self._AXIS_ORDERS, copy=False)
--> 989         return self._constructor(result, **d).__finalize__(self)
    990 
    991     # ideally we would define this to avoid the getattr checks, but

/usr/local/lib/python3.5/dist-packages/pandas/core/frame.py in __init__(self, data, index, columns, dtype, copy)
    304             else:
    305                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
--> 306                                          copy=copy)
    307         elif isinstance(data, (list, types.GeneratorType)):
    308             if isinstance(data, types.GeneratorType):

/usr/local/lib/python3.5/dist-packages/pandas/core/frame.py in _init_ndarray(self, values, index, columns, dtype, copy)
    481             values = maybe_infer_to_datetimelike(values)
    482 
--> 483         return create_block_manager_from_blocks([values], [columns, index])
    484 
    485     @property

/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in create_block_manager_from_blocks(blocks, axes)
   4301         blocks = [getattr(b, 'values', b) for b in blocks]
   4302         tot_items = sum(b.shape[0] for b in blocks)
-> 4303         construction_error(tot_items, blocks[0].shape[1:], axes, e)
   4304 
   4305 

/usr/local/lib/python3.5/dist-packages/pandas/core/internals.py in construction_error(tot_items, block_shape, axes, e)
   4278         raise ValueError("Empty data passed with indices specified.")
   4279     raise ValueError("Shape of passed values is {0}, indices imply {1}".format(
-> 4280         passed, implied))
   4281 
   4282 

ValueError: Shape of passed values is (1, 266), indices imply (1, 265)

[Granger7]

I'm having the exact same problem (discrepancy of exactly 1), but with a different size frame:

ValueError: Shape of passed values is (1, 13011), indices imply (0, 13010)

Did you have any luck fixing this, or anybody else have an idea?

[CptanPanic]

@Granger7 no I haven't figured it out, and it makes pyfolio unusable for me. Was hoping someone familiar with the code would have jumped in by now.

[eigenfoo]

Hi @CptanPanic, just a guess, but are you on the latest version of empyrical? There've been quite a few changes lately, so that might be the problem. If not, @twiecki should be able to help: the stack trace leads back to empyrical, which might be worrying.

[j220]

I am getting a similar error. Empyrical and all other dependencies are up to date. Return and benchmark series have definitely the same length. Traces back to sklearn. Any solutions to this issue?

ValueError                                Traceback (most recent call last)
<ipython-input-280-574b51b7a8af> in <module>()
      2 #benchmark_rets=test['returns'],
      3 
----> 4 pf.create_returns_tear_sheet(test['dnn_returns'], benchmark_rets=test['returns'], live_start_date=livestart)

~/anaconda/lib/python3.6/site-packages/pyfolio/plotting.py in call_w_context(*args, **kwargs)
     50         if set_context:
     51             with plotting_context(), axes_style():
---> 52                 return func(*args, **kwargs)
     53         else:
     54             return func(*args, **kwargs)

~/anaconda/lib/python3.6/site-packages/pyfolio/tears.py in create_returns_tear_sheet(returns, positions, transactions, live_start_date, cone_std, benchmark_rets, bootstrap, return_fig)
    551 
    552     plotting.plot_rolling_fama_french(
--> 553         returns, ax=ax_rolling_risk)
    554 
    555     # Drawdowns

~/anaconda/lib/python3.6/site-packages/pyfolio/plotting.py in plot_rolling_fama_french(returns, factor_returns, rolling_window, legend_loc, ax, **kwargs)
    188         returns,
    189         factor_returns=factor_returns,
--> 190         rolling_window=rolling_window)
    191 
    192     rolling_beta.plot(alpha=0.7, ax=ax, **kwargs)

~/anaconda/lib/python3.6/site-packages/pyfolio/timeseries.py in rolling_fama_french(returns, factor_returns, rolling_window)
    591                         factor_returns.index[rolling_window:]):
    592         coeffs = linear_model.LinearRegression().fit(factor_returns[beg:end],
--> 593                                                      returns[beg:end]).coef_
    594         regression_coeffs = np.append(regression_coeffs, [coeffs], axis=0)
    595 

~/anaconda/lib/python3.6/site-packages/sklearn/linear_model/base.py in fit(self, X, y, sample_weight)
    480         n_jobs_ = self.n_jobs
    481         X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
--> 482                          y_numeric=True, multi_output=True)
    483 
    484         if sample_weight is not None and np.atleast_1d(sample_weight).ndim > 1:

~/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py in check_X_y(X, y, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, warn_on_dtype, estimator)
    581         y = y.astype(np.float64)
    582 
--> 583     check_consistent_length(X, y)
    584 
    585     return X, y

~/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py in check_consistent_length(*arrays)
    202     if len(uniques) > 1:
    203         raise ValueError("Found input variables with inconsistent numbers of"
--> 204                          " samples: %r" % [int(l) for l in lengths])
    205 
    206 

ValueError: Found input variables with inconsistent numbers of samples: [127, 131]

[twiecki]

@CptanPanic Looks like you're passing in a dataframe where a Series is expected.

@j220 This should be fixed on master, can you update via: pip install -U --no-deps git+https://github.com/quantopian/alphalens

[j220]

Thanks for the help @twiecki. I updated but the error is still the same.

[twiecki]

Oops, I had an error above, should be: pip install -U --no-deps git+https://github.com/quantopian/pyfolio

[j220]

Now the error changes: ValueError: Input contains NaN, infinity or a value too large for dtype('float64').

[twiecki]

Can you post the full traceback?

[j220]

ValueError                                Traceback (most recent call last)
<ipython-input-92-df699bf1d804> in <module>()
      2 livestart='2017-01-01'
      3 #benchmark_rets=test['returns'],
----> 4 pf.create_returns_tear_sheet(test['dnn_returns'], benchmark_rets=test['returns'], live_start_date=livestart)

~/anaconda/lib/python3.6/site-packages/pyfolio/plotting.py in call_w_context(*args, **kwargs)
     50         if set_context:
     51             with plotting_context(), axes_style():
---> 52                 return func(*args, **kwargs)
     53         else:
     54             return func(*args, **kwargs)

~/anaconda/lib/python3.6/site-packages/pyfolio/tears.py in create_returns_tear_sheet(returns, positions, transactions, live_start_date, cone_std, benchmark_rets, bootstrap, turnover_denom, header_rows, return_fig)
    590 
    591     plotting.plot_rolling_fama_french(
--> 592         returns, ax=ax_rolling_risk)
    593 
    594     # Drawdowns

~/anaconda/lib/python3.6/site-packages/pyfolio/plotting.py in plot_rolling_fama_french(returns, factor_returns, rolling_window, legend_loc, ax, **kwargs)
    188         returns,
    189         factor_returns=factor_returns,
--> 190         rolling_window=rolling_window)
    191 
    192     rolling_beta.plot(alpha=0.7, ax=ax, **kwargs)

~/anaconda/lib/python3.6/site-packages/pyfolio/timeseries.py in rolling_regression(returns, factor_returns, rolling_window)
    596         reg = linear_model.LinearRegression(fit_intercept=True).fit(
    597             factor_returns.loc[returns_period.index],
--> 598             returns_period)
    599         rolling_risk.loc[end, factor_returns.columns] = reg.coef_
    600         rolling_risk.loc[end, 'alpha'] = reg.intercept_

~/anaconda/lib/python3.6/site-packages/sklearn/linear_model/base.py in fit(self, X, y, sample_weight)
    480         n_jobs_ = self.n_jobs
    481         X, y = check_X_y(X, y, accept_sparse=['csr', 'csc', 'coo'],
--> 482                          y_numeric=True, multi_output=True)
    483 
    484         if sample_weight is not None and np.atleast_1d(sample_weight).ndim > 1:

~/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py in check_X_y(X, y, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, multi_output, ensure_min_samples, ensure_min_features, y_numeric, warn_on_dtype, estimator)
    571     X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,
    572                     ensure_2d, allow_nd, ensure_min_samples,
--> 573                     ensure_min_features, warn_on_dtype, estimator)
    574     if multi_output:
    575         y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False,

~/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py in check_array(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)
    451                              % (array.ndim, estimator_name))
    452         if force_all_finite:
--> 453             _assert_all_finite(array)
    454 
    455     shape_repr = _shape_repr(array.shape)

~/anaconda/lib/python3.6/site-packages/sklearn/utils/validation.py in _assert_all_finite(X)
     42             and not np.isfinite(X).all()):
     43         raise ValueError("Input contains NaN, infinity"
---> 44                          " or a value too large for %r." % X.dtype)
     45 
     46 

ValueError: Input contains NaN, infinity or a value too large for dtype('float64').

[twiecki]

Do you have nans in your returns / factor_returns?

[j220]

Double checked, no nans or inf in any series. Maybe an issue with the frequency of my data in business days? data=(data.resample('B').last())

[gusgordon]

The original issue here is separate from the latest issue you guys brought up — the latest issue is related to this: https://github.com/quantopian/pyfolio/issues/462.

I'll put in a fix for this, it's currently blocking me as well.

edit: https://github.com/quantopian/pyfolio/pull/464

[hardikec2003]

try np.squeeze(). It may work.

[sshojiro]

Just FYI, nobody has mentioned; but this problem seems to be caused from a numpy bug, especially in np.delete. I encountered a quite similar error when using numpy.

And I avoid this error with the code in this link: https://stackoverflow.com/questions/19286657/index-all-except-one-item-in-python?utm_medium=organic&utm_source=google_rich_qa&utm_campaign=google_rich_qa

[sairamdgr8]

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in create_block_manager_from_blocks(blocks, axes)
   4621                 blocks = [make_block(values=blocks[0],
-> 4622                                      placement=slice(0, len(axes[0])))]
   4623 

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in make_block(values, placement, klass, ndim, dtype, fastpath)
   2956 
-> 2957     return klass(values, ndim=ndim, fastpath=fastpath, placement=placement)
   2958 

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in __init__(self, values, placement, ndim, fastpath)
    119                              'implies %d' % (len(self.values),
--> 120                                              len(self.mgr_locs)))
    121 

ValueError: Wrong number of items passed 5, placement implies 1

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-26-261beb8fa7e1> in <module>()
----> 1 ddf=pd.DataFrame(lm.coef_,X.columns,columns=['coeffs'])

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py in __init__(self, data, index, columns, dtype, copy)
    359             else:
    360                 mgr = self._init_ndarray(data, index, columns, dtype=dtype,
--> 361                                          copy=copy)
    362         elif isinstance(data, (list, types.GeneratorType)):
    363             if isinstance(data, types.GeneratorType):

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\frame.py in _init_ndarray(self, values, index, columns, dtype, copy)
    531             values = maybe_infer_to_datetimelike(values)
    532 
--> 533         return create_block_manager_from_blocks([values], [columns, index])
    534 
    535     @property

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in create_block_manager_from_blocks(blocks, axes)
   4629         blocks = [getattr(b, 'values', b) for b in blocks]
   4630         tot_items = sum(b.shape[0] for b in blocks)
-> 4631         construction_error(tot_items, blocks[0].shape[1:], axes, e)
   4632 
   4633 

C:\ProgramData\Anaconda3\lib\site-packages\pandas\core\internals.py in construction_error(tot_items, block_shape, axes, e)
   4606         raise ValueError("Empty data passed with indices specified.")
   4607     raise ValueError("Shape of passed values is {0}, indices imply {1}".format(
-> 4608         passed, implied))
   4609 
   4610 

ValueError: Shape of passed values is (5, 1), indices imply (1, 5)

[hadesophio]

Assignment the dataframe to another variable and run the code