INFO:root:begin fit
INFO:root:Start training with [gpu(6), gpu(7)]
iter
Traceback (most recent call last):
File "testcrnn.py", line 249, in
epoch_end_callback = mx.callback.do_checkpoint(prefix, 1))
File "../../python/mxnet/model.py", line 811, in fit
sym_gen=self.sym_gen)
File "../../python/mxnet/model.py", line 259, in _train_multi_device
executor_manager.update_metric(eval_metric, data_batch.label)
File "../../python/mxnet/executor_manager.py", line 422, in update_metric
self.curr_execgrp.update_metric(metric, labels)
File "../../python/mxnet/executor_manager.py", line 274, in update_metric
metric.update(labels_slice, texec.outputs)
File "../../python/mxnet/metric.py", line 350, in update
reval = self._feval(label, pred)
File "../../python/mxnet/metric.py", line 379, in feval
return numpy_feval(label, pred)
File "testcrnn.py", line 166, in Accuracy
p.append(np.argmax(pred[k * BATCH_SIZE + i]))
IndexError: index 1600 is out of bounds for axis 0 with size 1600
INFO:root:begin fit INFO:root:Start training with [gpu(6), gpu(7)] iter Traceback (most recent call last): File "testcrnn.py", line 249, in
epoch_end_callback = mx.callback.do_checkpoint(prefix, 1))
File "../../python/mxnet/model.py", line 811, in fit
sym_gen=self.sym_gen)
File "../../python/mxnet/model.py", line 259, in _train_multi_device
executor_manager.update_metric(eval_metric, data_batch.label)
File "../../python/mxnet/executor_manager.py", line 422, in update_metric
self.curr_execgrp.update_metric(metric, labels)
File "../../python/mxnet/executor_manager.py", line 274, in update_metric
metric.update(labels_slice, texec.outputs)
File "../../python/mxnet/metric.py", line 350, in update
reval = self._feval(label, pred)
File "../../python/mxnet/metric.py", line 379, in feval
return numpy_feval(label, pred)
File "testcrnn.py", line 166, in Accuracy
p.append(np.argmax(pred[k * BATCH_SIZE + i]))
IndexError: index 1600 is out of bounds for axis 0 with size 1600