Error while training

[fredO13]

Had a problem while training on my own dataset converted from coco to icdar_2013.

here is the output :

Looking for blah\craft_mlt_25k.h5

blah\tensorflow\python\keras\engine\training.py:1844: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
  warnings.warn('`Model.fit_generator` is deprecated and '

Epoch 1/1000
11/90 [==>...........................] - ETA: 1:01 - loss: 5.4118e-04

---------------------------------------------------------------------------
UnknownError                              Traceback (most recent call last)
<ipython-input-9-a78793f98407> in <module>()
     19     ],
     20     validation_data=validation_generator,
---> 21     validation_steps=math.ceil(len(validation) / batch_size)
     22 )

blah\tensorflow\python\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   1859         use_multiprocessing=use_multiprocessing,
   1860         shuffle=shuffle,
-> 1861         initial_epoch=initial_epoch)
   1862 
   1863   def evaluate_generator(self,

blah\tensorflow\python\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
   1098                 _r=1):
   1099               callbacks.on_train_batch_begin(step)
-> 1100               tmp_logs = self.train_function(iterator)
   1101               if data_handler.should_sync:
   1102                 context.async_wait()

blah\tensorflow\python\eager\def_function.py in __call__(self, *args, **kwds)
    826     tracing_count = self.experimental_get_tracing_count()
    827     with trace.Trace(self._name) as tm:
--> 828       result = self._call(*args, **kwds)
    829       compiler = "xla" if self._experimental_compile else "nonXla"
    830       new_tracing_count = self.experimental_get_tracing_count()

blah\tensorflow\python\eager\def_function.py in _call(self, *args, **kwds)
    853       # In this case we have created variables on the first call, so we run the
    854       # defunned version which is guaranteed to never create variables.
--> 855       return self._stateless_fn(*args, **kwds)  # pylint: disable=not-callable
    856     elif self._stateful_fn is not None:
    857       # Release the lock early so that multiple threads can perform the call

blah\tensorflow\python\eager\function.py in __call__(self, *args, **kwargs)
   2941        filtered_flat_args) = self._maybe_define_function(args, kwargs)
   2942     return graph_function._call_flat(
-> 2943         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access
   2944 
   2945   @property

blah\tensorflow\python\eager\function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
   1917       # No tape is watching; skip to running the function.
   1918       return self._build_call_outputs(self._inference_function.call(
-> 1919           ctx, args, cancellation_manager=cancellation_manager))
   1920     forward_backward = self._select_forward_and_backward_functions(
   1921         args,

blah\tensorflow\python\eager\function.py in call(self, ctx, args, cancellation_manager)
    558               inputs=args,
    559               attrs=attrs,
--> 560               ctx=ctx)
    561         else:
    562           outputs = execute.execute_with_cancellation(

blah\tensorflow\python\eager\execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     58     ctx.ensure_initialized()
     59     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60                                         inputs, attrs, num_outputs)
     61   except core._NotOkStatusException as e:
     62     if name is not None:

UnknownError: 2 root error(s) found.
  (0) Unknown:  IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed
Traceback (most recent call last):

  File "blah\tensorflow\python\ops\script_ops.py", line 249, in __call__
    ret = func(*args)

  File "blah\tensorflow\python\autograph\impl\api.py", line 620, in wrapper
    return func(*args, **kwargs)

  File "blah\tensorflow\python\data\ops\dataset_ops.py", line 891, in generator_py_func
    values = next(generator_state.get_iterator(iterator_id))

  File "blah\tensorflow\python\keras\engine\data_adapter.py", line 807, in wrapped_generator
    for data in generator_fn():

  File "blah\keras_ocr\detection.py", line 642, in get_batch_generator
    lines=lines) for lines in line_groups

  File "blah\keras_ocr\detection.py", line 642, in <listcomp>
    lines=lines) for lines in line_groups

  File "blah\keras_ocr\detection.py", line 107, in compute_maps
    line, orientation = tools.fix_line(line)

  File "blah\keras_ocr\tools.py", line 520, in fix_line
    sortedx = centers[:, 0].argsort()

IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed

     [[{{node PyFunc}}]]
     [[IteratorGetNext]]
  (1) Unknown:  IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed
Traceback (most recent call last):

  File "blah\tensorflow\python\ops\script_ops.py", line 249, in __call__
    ret = func(*args)

  File "blah\tensorflow\python\autograph\impl\api.py", line 620, in wrapper
    return func(*args, **kwargs)

  File "blah\tensorflow\python\data\ops\dataset_ops.py", line 891, in generator_py_func
    values = next(generator_state.get_iterator(iterator_id))

  File "blah\tensorflow\python\keras\engine\data_adapter.py", line 807, in wrapped_generator
    for data in generator_fn():

  File "blah\keras_ocr\detection.py", line 642, in get_batch_generator
    lines=lines) for lines in line_groups

  File "blah\keras_ocr\detection.py", line 642, in <listcomp>
    lines=lines) for lines in line_groups

  File "blah\keras_ocr\detection.py", line 107, in compute_maps
    line, orientation = tools.fix_line(line)

  File "blah\keras_ocr\tools.py", line 520, in fix_line
    sortedx = centers[:, 0].argsort()

IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed

     [[{{node PyFunc}}]]
     [[IteratorGetNext]]
     [[IteratorGetNext/_4]]
0 successful operations.
0 derived errors ignored. [Op:__inference_train_function_7457]

Function call stack:
train_function -> train_function

I updated to the lastest code but got the same error... the difference with icdar is that my characters are defined in quadrangles, not axis-aligned boxes... could that be the issue ?

[fredO13]

Sorry, I found out my dataset has some missing data. I fixed it