MXNet error when running `make_evaluation_predictions`

kurvaraviteja355 commented 2 years ago

Description

I got the following error while making the make_evaluation_predictions

surface_dates = [pd.Timestamp("2017-01-01", freq='D') for _ in range(len(df))]

train_ds = ListDataset([
    {
        FieldName.TARGET: target,
        FieldName.START: start,
        FieldName.FEAT_DYNAMIC_REAL: fdr,
        FieldName.FEAT_STATIC_CAT: fsc
    }
    for (target, start, fdr, fsc) in zip(train_target_values,
                                        surface_dates,
                                        train_cal_features_list,
                                        stat_cat)
], freq="D")

test_ds = ListDataset([
    {
        FieldName.TARGET: target,
        FieldName.START: start,
        FieldName.FEAT_DYNAMIC_REAL: fdr,
        FieldName.FEAT_STATIC_CAT: fsc
    }
    for (target, start, fdr, fsc) in zip(test_target_values,
                                        surface_dates,
                                        test_cal_features_list,
                                        stat_cat)
], freq="D")

estimator = DeepAREstimator(
    prediction_length=validation_prediction_length,
    freq="D",
    distr_output = NegativeBinomialOutput(),
    use_feat_dynamic_real=True,
    use_feat_static_cat=True,
    cardinality=stat_cat_cardinalities,
    trainer=Trainer(
        learning_rate=1e-3,
        epochs=3,
        num_batches_per_epoch=50,
        batch_size=32
    )
)
predictor = estimator.train(train_ds)

forecast_it, ts_it = make_evaluation_predictions(
    dataset=test_ds,
    predictor=predictor,
    num_samples=5
    )

print("Obtaining time series conditioning values ...")
tss = list(tqdm(ts_it, total=len(test_ds)))
print("Obtaining time series predictions ...")
forecasts = list(tqdm(forecast_it, total=len(test_ds)))

Error message or code output

(Paste the complete error message, including stack trace, or the undesired output that the above snippet produces.)

Obtaining time series conditioning values ...
100%|██████████| 1965/1965 [00:13<00:00, 149.79it/s]
Obtaining time series predictions ...
  0%|          | 0/1965 [00:01<?, ?it/s]
---------------------------------------------------------------------------
MXNetError                                Traceback (most recent call last)
~\AppData\Local\Temp\ipykernel_31452\4056093787.py in <module>
      8 tss = list(tqdm(ts_it, total=len(test_ds)))
      9 print("Obtaining time series predictions ...")
---> 10 forecasts = list(tqdm(forecast_it, total=len(test_ds)))

~\Anaconda3\envs\Glounts\lib\site-packages\tqdm\std.py in __iter__(self)
   1193 
   1194         try:
-> 1195             for obj in iterable:
   1196                 yield obj
   1197                 # Update and possibly print the progressbar.

~\Anaconda3\envs\Glounts\lib\site-packages\gluonts\mx\model\predictor.py in predict(self, dataset, num_samples, num_workers, num_prefetch, **kwargs)
    168                 freq=self.freq,
    169                 output_transform=self.output_transform,
--> 170                 num_samples=num_samples,
    171             )
    172 

~\Anaconda3\envs\Glounts\lib\site-packages\gluonts\model\forecast_generator.py in __call__(self, inference_data_loader, prediction_net, input_names, freq, output_transform, num_samples, **kwargs)
    172         for batch in inference_data_loader:
    173             inputs = [batch[k] for k in input_names]
--> 174             outputs = predict_to_numpy(prediction_net, inputs)
    175             if output_transform is not None:
    176                 outputs = output_transform(batch, outputs)

~\Anaconda3\envs\Glounts\lib\functools.py in wrapper(*args, **kw)
    838                             '1 positional argument')
    839 
--> 840         return dispatch(args[0].__class__)(*args, **kw)
    841 
    842     funcname = getattr(func, '__name__', 'singledispatch function')

~\Anaconda3\envs\Glounts\lib\site-packages\gluonts\mx\model\predictor.py in _(prediction_net, inputs)
     47 @predict_to_numpy.register(mx.gluon.Block)
     48 def _(prediction_net: mx.gluon.Block, inputs: mx.ndarray) -> np.ndarray:
---> 49     return prediction_net(*inputs).asnumpy()
     50 
     51 

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\gluon\block.py in __call__(self, *args)
    680             hook(self, args)
    681 
--> 682         out = self.forward(*args)
    683 
    684         for hook in self._forward_hooks.values():

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\gluon\block.py in forward(self, x, *args)
   1252                     params = {k: v.data(ctx) for k, v in self._reg_params.items()}
   1253 
-> 1254                 return self.hybrid_forward(ndarray, x, *args, **params)
   1255         params = {i: j.var() for i, j in self._reg_params.items()}
   1256         with self.name_scope():

~\Anaconda3\envs\Glounts\lib\site-packages\gluonts\model\deepar\_network.py in hybrid_forward(self, F, feat_static_cat, feat_static_real, past_time_feat, past_target, past_observed_values, future_time_feat, past_is_pad)
   1160             static_feat=static_feat,
   1161             scale=scale,
-> 1162             begin_states=state,
   1163         )

~\Anaconda3\envs\Glounts\lib\site-packages\gluonts\model\deepar\_network.py in sampling_decoder(self, F, static_feat, past_target, time_feat, scale, begin_states)
   1067             decoder_input = F.concat(
   1068                 input_lags,
-> 1069                 repeated_time_feat.slice_axis(axis=1, begin=k, end=k + 1),
   1070                 # observed_values.expand_dims(axis=1),
   1071                 repeated_static_feat,

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\ndarray\ndarray.py in slice_axis(self, *args, **kwargs)
   1606         this array as data.
   1607         """
-> 1608         return op.slice_axis(self, *args, **kwargs)
   1609 
   1610     def slice_like(self, *args, **kwargs):

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\ndarray\register.py in slice_axis(data, axis, begin, end, out, name, **kwargs)

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\_ctypes\ndarray.py in _imperative_invoke(handle, ndargs, keys, vals, out, is_np_op, output_is_list)
     89         c_str_array(keys),
     90         c_str_array([str(s) for s in vals]),
---> 91         ctypes.byref(out_stypes)))
     92 
     93     create_ndarray_fn = _global_var._np_ndarray_cls if is_np_op else _global_var._ndarray_cls

~\Anaconda3\envs\Glounts\lib\site-packages\mxnet\base.py in check_call(ret)
    244     """
    245     if ret != 0:
--> 246         raise get_last_ffi_error()
    247 
    248 

MXNetError: Traceback (most recent call last):
  File "C:\Jenkins\workspace\mxnet-tag\mxnet\src\c_api\c_api_ndarray.cc", line 60
MXNetError: Check failed: inp->shape().Size() < (int64_t{1} << 31) - 1 (3062188800 vs. 2147483647) : [SetNDInputsOutputs] Size of tensor you are trying to allocate is larger than 2^31 elements. Please build with flag USE_INT64_TENSOR_SIZE=1

Environment

Operating system: Windows 10
Python version: 3.7
GluonTS version:0.9.2
MXNet version:1.7

(Add as much information about your environment as possible, e.g. dependencies versions.)

lostella commented 2 years ago

@kurvaraviteja355 thanks for filing the issue! I think your code snippet won’t run since some definitions are not included. Is there any way you can reproduce the error with a minimal example? (Eg with a small random dataset)

kurvaraviteja355 commented 2 years ago

Still i got the same error with small random dataset.

I have multiple time series data (for last 5 years) and i need to forecast the values for next 6 months i.e 180 days in future and i have in total of 393 stores with5 different products in each store

lostella commented 2 years ago

@kurvaraviteja355 got it. Could you share a self-contained snippet (with random data) that reproduces the error? Otherwise it's hard to say what may be going wrong.

awslabs / gluonts