Issues with regex on Windows

[safusu]

I upgraded the pip package via

pip install keras-video-generators -U and re-ran my code, got the following error:

class fast, validation count: 20, test count: 16, train count: 68
class medi, validation count: 251, test count: 201, train count: 804
class null, validation count: 138, test count: 110, train count: 443
class slow, validation count: 499, test count: 399, train count: 1600
Total data: 4 classes for 2915 files for train
Checking files to find possible sequences, please wait...
For 2915 files, I found 20317 possible sequence samples
Total data: 4 classes for 908 files for validation
Checking files to find possible sequences, please wait...
For 908 files, I found 6367 possible sequence samples
Epoch 1/50
---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
<ipython-input-5-d4f14ca27752> in <module>
     78         verbose=1,
     79         epochs=EPOCHS,
---> 80         callbacks=callbacks
     81     )
     82 

~\Anaconda3\envs\research\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

~\Anaconda3\envs\research\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   1730             use_multiprocessing=use_multiprocessing,
   1731             shuffle=shuffle,
-> 1732             initial_epoch=initial_epoch)
   1733 
   1734     @interfaces.legacy_generator_methods_support

~\Anaconda3\envs\research\lib\site-packages\keras\engine\training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
    183             batch_index = 0
    184             while steps_done < steps_per_epoch:
--> 185                 generator_output = next(output_generator)
    186 
    187                 if not hasattr(generator_output, '__len__'):

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get(self)
    623         except Exception:
    624             self.stop()
--> 625             six.reraise(*sys.exc_info())
    626 
    627 

~\Anaconda3\envs\research\lib\site-packages\six.py in reraise(tp, value, tb)
    701             if value.__traceback__ is not tb:
    702                 raise value.with_traceback(tb)
--> 703             raise value
    704         finally:
    705             value = None

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get(self)
    608                 try:
    609                     future = self.queue.get(block=True)
--> 610                     inputs = future.get(timeout=30)
    611                 except mp.TimeoutError:
    612                     idx = future.idx

~\Anaconda3\envs\research\lib\multiprocessing\pool.py in get(self, timeout)
    655             return self._value
    656         else:
--> 657             raise self._value
    658 
    659     def _set(self, i, obj):

~\Anaconda3\envs\research\lib\multiprocessing\pool.py in worker(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)
    119         job, i, func, args, kwds = task
    120         try:
--> 121             result = (True, func(*args, **kwds))
    122         except Exception as e:
    123             if wrap_exception and func is not _helper_reraises_exception:

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get_index(uid, i)
    404         The value at index `i`.
    405     """
--> 406     return _SHARED_SEQUENCES[uid][i]
    407 
    408 

~\Anaconda3\envs\research\lib\site-packages\keras_video\sliding.py in __getitem__(self, idx)
    157             vid = self.vid_info[i]
    158             video = vid.get('name')
--> 159             classname = self._get_classname(video)
    160 
    161             # create a label array and set 1 to the right column

~\Anaconda3\envs\research\lib\site-packages\keras_video\generator.py in _get_classname(self, video)
    328         for src, dest in RE_PATH_REPLACE.items():
    329             pattern = pattern.replace(src, dest)
--> 330         classname = re.findall(pattern, video)[0]
    331         return classname
    332 

~\Anaconda3\envs\research\lib\re.py in findall(pattern, string, flags)
    221 
    222     Empty matches are included in the result."""
--> 223     return _compile(pattern, flags).findall(string)
    224 
    225 def finditer(pattern, string, flags=0):

~\Anaconda3\envs\research\lib\re.py in _compile(pattern, flags)
    284     if not sre_compile.isstring(pattern):
    285         raise TypeError("first argument must be string or compiled pattern")
--> 286     p = sre_compile.compile(pattern, flags)
    287     if not (flags & DEBUG):
    288         if len(_cache) >= _MAXCACHE:

~\Anaconda3\envs\research\lib\sre_compile.py in compile(p, flags)
    762     if isstring(p):
    763         pattern = p
--> 764         p = sre_parse.parse(p, flags)
    765     else:
    766         pattern = None

~\Anaconda3\envs\research\lib\sre_parse.py in parse(str, flags, pattern)
    922 
    923     try:
--> 924         p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0)
    925     except Verbose:
    926         # the VERBOSE flag was switched on inside the pattern.  to be

~\Anaconda3\envs\research\lib\sre_parse.py in _parse_sub(source, state, verbose, nested)
    418     while True:
    419         itemsappend(_parse(source, state, verbose, nested + 1,
--> 420                            not nested and not items))
    421         if not sourcematch("|"):
    422             break

~\Anaconda3\envs\research\lib\sre_parse.py in _parse(source, state, verbose, nested, first)
    499 
    500         if this[0] == "\\":
--> 501             code = _escape(source, this, state)
    502             subpatternappend(code)
    503 

~\Anaconda3\envs\research\lib\sre_parse.py in _escape(source, escape, state)
    367             escape += source.getwhile(8, HEXDIGITS)
    368             if len(escape) != 10:
--> 369                 raise source.error("incomplete escape %s" % escape, len(escape))
    370             c = int(escape[2:], 16)
    371             chr(c) # raise ValueError for invalid code

error: incomplete escape \U at position 2

It looks like it breaks at the classname = re.findall(pattern, video)[0] line: the string for pattern and video are as follows:

pattern: C:\Users\Jeren\Desktop\6_solarmax_2000_2003_2012_2015\(.*?)\.*\.mp4 video: C:\Users\Jeren\Desktop\6_solarmax_2000_2003_2012_2015\slow\20150308.113605.mp4

I haven't used the re package in python before, so I'm trying out some possible solutions, but I'm pretty sure this is one of those Windows-only path problems.

[AdityaGandotra]

Hi mate, defining a variable verbose =0 before fitting the model did the trick for me.

[metal3d]

Hello, OK I see the problem. Windows is definitively getting me crazy by using \ as path separator. The problem is that C:\Users contains '\Uthat is a prefix to find unicode char. So, I will need to make a fix to replace Windows path separators before to send it tore`.

Thanks a lot for that issue !

[Mitesh1999]

I upgraded the pip package via

pip install keras-video-generators -U and re-ran my code, got the following error:

class fast, validation count: 20, test count: 16, train count: 68
class medi, validation count: 251, test count: 201, train count: 804
class null, validation count: 138, test count: 110, train count: 443
class slow, validation count: 499, test count: 399, train count: 1600
Total data: 4 classes for 2915 files for train
Checking files to find possible sequences, please wait...
For 2915 files, I found 20317 possible sequence samples
Total data: 4 classes for 908 files for validation
Checking files to find possible sequences, please wait...
For 908 files, I found 6367 possible sequence samples
Epoch 1/50
---------------------------------------------------------------------------
error                                     Traceback (most recent call last)
<ipython-input-5-d4f14ca27752> in <module>
     78         verbose=1,
     79         epochs=EPOCHS,
---> 80         callbacks=callbacks
     81     )
     82 

~\Anaconda3\envs\research\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

~\Anaconda3\envs\research\lib\site-packages\keras\engine\training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   1730             use_multiprocessing=use_multiprocessing,
   1731             shuffle=shuffle,
-> 1732             initial_epoch=initial_epoch)
   1733 
   1734     @interfaces.legacy_generator_methods_support

~\Anaconda3\envs\research\lib\site-packages\keras\engine\training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
    183             batch_index = 0
    184             while steps_done < steps_per_epoch:
--> 185                 generator_output = next(output_generator)
    186 
    187                 if not hasattr(generator_output, '__len__'):

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get(self)
    623         except Exception:
    624             self.stop()
--> 625             six.reraise(*sys.exc_info())
    626 
    627 

~\Anaconda3\envs\research\lib\site-packages\six.py in reraise(tp, value, tb)
    701             if value.__traceback__ is not tb:
    702                 raise value.with_traceback(tb)
--> 703             raise value
    704         finally:
    705             value = None

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get(self)
    608                 try:
    609                     future = self.queue.get(block=True)
--> 610                     inputs = future.get(timeout=30)
    611                 except mp.TimeoutError:
    612                     idx = future.idx

~\Anaconda3\envs\research\lib\multiprocessing\pool.py in get(self, timeout)
    655             return self._value
    656         else:
--> 657             raise self._value
    658 
    659     def _set(self, i, obj):

~\Anaconda3\envs\research\lib\multiprocessing\pool.py in worker(inqueue, outqueue, initializer, initargs, maxtasks, wrap_exception)
    119         job, i, func, args, kwds = task
    120         try:
--> 121             result = (True, func(*args, **kwds))
    122         except Exception as e:
    123             if wrap_exception and func is not _helper_reraises_exception:

~\Anaconda3\envs\research\lib\site-packages\keras\utils\data_utils.py in get_index(uid, i)
    404         The value at index `i`.
    405     """
--> 406     return _SHARED_SEQUENCES[uid][i]
    407 
    408 

~\Anaconda3\envs\research\lib\site-packages\keras_video\sliding.py in __getitem__(self, idx)
    157             vid = self.vid_info[i]
    158             video = vid.get('name')
--> 159             classname = self._get_classname(video)
    160 
    161             # create a label array and set 1 to the right column

~\Anaconda3\envs\research\lib\site-packages\keras_video\generator.py in _get_classname(self, video)
    328         for src, dest in RE_PATH_REPLACE.items():
    329             pattern = pattern.replace(src, dest)
--> 330         classname = re.findall(pattern, video)[0]
    331         return classname
    332 

~\Anaconda3\envs\research\lib\re.py in findall(pattern, string, flags)
    221 
    222     Empty matches are included in the result."""
--> 223     return _compile(pattern, flags).findall(string)
    224 
    225 def finditer(pattern, string, flags=0):

~\Anaconda3\envs\research\lib\re.py in _compile(pattern, flags)
    284     if not sre_compile.isstring(pattern):
    285         raise TypeError("first argument must be string or compiled pattern")
--> 286     p = sre_compile.compile(pattern, flags)
    287     if not (flags & DEBUG):
    288         if len(_cache) >= _MAXCACHE:

~\Anaconda3\envs\research\lib\sre_compile.py in compile(p, flags)
    762     if isstring(p):
    763         pattern = p
--> 764         p = sre_parse.parse(p, flags)
    765     else:
    766         pattern = None

~\Anaconda3\envs\research\lib\sre_parse.py in parse(str, flags, pattern)
    922 
    923     try:
--> 924         p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0)
    925     except Verbose:
    926         # the VERBOSE flag was switched on inside the pattern.  to be

~\Anaconda3\envs\research\lib\sre_parse.py in _parse_sub(source, state, verbose, nested)
    418     while True:
    419         itemsappend(_parse(source, state, verbose, nested + 1,
--> 420                            not nested and not items))
    421         if not sourcematch("|"):
    422             break

~\Anaconda3\envs\research\lib\sre_parse.py in _parse(source, state, verbose, nested, first)
    499 
    500         if this[0] == "\\":
--> 501             code = _escape(source, this, state)
    502             subpatternappend(code)
    503 

~\Anaconda3\envs\research\lib\sre_parse.py in _escape(source, escape, state)
    367             escape += source.getwhile(8, HEXDIGITS)
    368             if len(escape) != 10:
--> 369                 raise source.error("incomplete escape %s" % escape, len(escape))
    370             c = int(escape[2:], 16)
    371             chr(c) # raise ValueError for invalid code

error: incomplete escape \U at position 2

It looks like it breaks at the classname = re.findall(pattern, video)[0] line: the string for pattern and video are as follows:

pattern: C:\Users\Jeren\Desktop\6_solarmax_2000_2003_2012_2015\(.*?)\.*\.mp4 video: C:\Users\Jeren\Desktop\6_solarmax_2000_2003_2012_2015\slow\20150308.113605.mp4

I haven't used the re package in python before, so I'm trying out some possible solutions, but I'm pretty sure this is one of those Windows-only path problems.

I am getting error: "invalid group reference 8" which is connected with re. I am running the example from github on Windows machine. My glob_pattern is "D:\8th Sem\videos\{classname}\*.avi".

Error Stack Trace:

runfile('D:/8th Sem/mvideos.py', wdir='D:/8th Sem') ['cartwheel', 'catch', 'fencing'] class cartwheel, validation count: 10, train count: 97 class catch, validation count: 10, train count: 92 class fencing, validation count: 11, train count: 105 Total data: 3 classes for 294 files for train Total data: 3 classes for 31 files for validation 4 Epoch 1/50 Traceback (most recent call last):

File "D:\8th Sem\mvideos.py", line 149, in callbacks=callbacks

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\legacy\interfaces.py", line 91, in wrapper return func(*args, **kwargs)

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training.py", line 1732, in fit_generator initial_epoch=initial_epoch)

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\engine\training_generator.py", line 185, in fit_generator generator_output = next(output_generator)

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\data_utils.py", line 625, in get six.reraise(*sys.exc_info())

File "C:\ProgramData\Anaconda3\lib\site-packages\six.py", line 703, in reraise raise value

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\data_utils.py", line 610, in get inputs = future.get(timeout=30)

File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 657, in get raise self._value

File "C:\ProgramData\Anaconda3\lib\multiprocessing\pool.py", line 121, in worker result = (True, func(*args, **kwds))

File "C:\ProgramData\Anaconda3\lib\site-packages\keras\utils\data_utils.py", line 406, in get_index return _SHARED_SEQUENCES[uid][i]

File "C:\ProgramData\Anaconda3\lib\site-packages\keras_video\generator.py", line 298, in getitem classname = self._get_classname(video)

File "C:\ProgramData\Anaconda3\lib\site-packages\keras_video\generator.py", line 327, in _get_classname length = len(re.findall(pattern,video))

File "C:\ProgramData\Anaconda3\lib\re.py", line 223, in findall return _compile(pattern, flags).findall(string)

File "C:\ProgramData\Anaconda3\lib\re.py", line 286, in _compile p = sre_compile.compile(pattern, flags)

File "C:\ProgramData\Anaconda3\lib\sre_compile.py", line 764, in compile p = sre_parse.parse(p, flags)

File "C:\ProgramData\Anaconda3\lib\sre_parse.py", line 930, in parse p = _parse_sub(source, pattern, flags & SRE_FLAG_VERBOSE, 0)

File "C:\ProgramData\Anaconda3\lib\sre_parse.py", line 426, in _parse_sub not nested and not items))

File "C:\ProgramData\Anaconda3\lib\sre_parse.py", line 507, in _parse code = _escape(source, this, state)

File "C:\ProgramData\Anaconda3\lib\sre_parse.py", line 399, in _escape raise source.error("invalid group reference %d" % group, len(escape) - 1)

error: invalid group reference 8

[metal3d]

OK... I've installed a "modern.ie" VM to find a way to fix that. I did it in commit 207dc93 but there is problems with path like "./" and so on...

If you're confortable to use the git version, so you can try the fix. Or you can try to copy the REGEXP variable I set up in generator.py (on top).

Anyway, I will produce a new package version as soon as I find a proper way to use pathutils or something like that.

[metal3d]

Seems like I found a proper way to avoid the problem, it's in master branch now. I also added tests and an classes autodiscovery.

The next release will fix that.

[safusu]

I cloned the repo and built the library files - they work great now, no errors! Thank you so much for your ongoing support~

[metal3d]

OK, thanks a lot for your tests.

I will make the new release in a couple of minutes/hours, finishing the unit tests and Tavis config.