search

NLeSC / mcfly-tutorial

tutorial for mcfly repository
Apache License 2.0
77 stars 40 forks source link

Loading model does not work #28

Open egpbos opened 7 years ago

egpbos commented 7 years ago

In the tutorial, loading the pre-trained model does not work.

model = load_model('./model/model.h5')

Gives:

---------------------------------------------------------------------------
SystemError                               Traceback (most recent call last)
<ipython-input-26-5d87631adf41> in <module>()
----> 1 model = load_model('./model/model.h5')

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/models.py in load_model(filepath, custom_objects, compile)
    237             raise ValueError('No model found in config file.')
    238         model_config = json.loads(model_config.decode('utf-8'))
--> 239         model = model_from_config(model_config, custom_objects=custom_objects)
    240 
    241         # set weights

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/models.py in model_from_config(config, custom_objects)
    311                         'Maybe you meant to use '
    312                         '`Sequential.from_config(config)`?')
--> 313     return layer_module.deserialize(config, custom_objects=custom_objects)
    314 
    315 

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/layers/__init__.py in deserialize(config, custom_objects)
     52                                     module_objects=globs,
     53                                     custom_objects=custom_objects,
---> 54                                     printable_module_name='layer')

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/utils/generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    137                 return cls.from_config(config['config'],
    138                                        custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 139                                                            list(custom_objects.items())))
    140             with CustomObjectScope(custom_objects):
    141                 return cls.from_config(config['config'])

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/models.py in from_config(cls, config, custom_objects)
   1212         for conf in config:
   1213             layer = layer_module.deserialize(conf, custom_objects=custom_objects)
-> 1214             model.add(layer)
   1215         return model
   1216 

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/models.py in add(self, layer)
    473                           output_shapes=[self.outputs[0]._keras_shape])
    474         else:
--> 475             output_tensor = layer(self.outputs[0])
    476             if isinstance(output_tensor, list):
    477                 raise TypeError('All layers in a Sequential model '

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/engine/topology.py in __call__(self, inputs, **kwargs)
    600 
    601             # Actually call the layer, collecting output(s), mask(s), and shape(s).
--> 602             output = self.call(inputs, **kwargs)
    603             output_mask = self.compute_mask(inputs, previous_mask)
    604 

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/layers/core.py in call(self, inputs, mask)
    648         if has_arg(self.function, 'mask'):
    649             arguments['mask'] = mask
--> 650         return self.function(inputs, **arguments)
    651 
    652     def compute_mask(self, inputs, mask=None):

~/sw/miniconda3/envs/mcfly-tutorial/lib/python3.6/site-packages/keras/layers/core.py in <lambda>(x)
    176 
    177     # Input shape
--> 178         4D tensor with shape:
    179         `(samples, channels, rows, cols)` if data_format='channels_first'
    180         or 4D tensor with shape:

SystemError: unknown opcode

This is on macOS, with conda Python 3.6.2. pip freeze gives the following list:

appnope==0.1.0
bleach==1.5.0
certifi==2016.2.28
cycler==0.10.0
decorator==4.1.2
entrypoints==0.2.3
h5py==2.7.0
html5lib==0.9999999
ipykernel==4.6.1
ipython==6.1.0
ipython-genutils==0.2.0
ipywidgets==6.0.0
jedi==0.10.2
Jinja2==2.9.6
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.1.0
jupyter-console==5.2.0
jupyter-core==4.3.0
Keras==2.0.8
Markdown==2.6.9
MarkupSafe==1.0
matplotlib==2.0.2
mcfly==1.0.1
mistune==0.7.4
nbconvert==5.2.1
nbformat==4.4.0
notebook==5.0.0
numpy==1.13.1
pandas==0.20.3
pandocfilters==1.4.2
pexpect==4.2.1
pickleshare==0.7.4
prompt-toolkit==1.0.15
protobuf==3.4.0
ptyprocess==0.5.2
Pygments==2.2.0
pyparsing==2.2.0
python-dateutil==2.6.1
pytz==2017.2
PyYAML==3.12
pyzmq==16.0.2
qtconsole==4.3.1
scikit-learn==0.19.0
scipy==0.19.1
simplegeneric==0.8.1
six==1.10.0
tensorflow==1.3.0
tensorflow-tensorboard==0.1.6
terminado==0.6
testpath==0.3
tornado==4.5.2
traitlets==4.3.2
wcwidth==0.1.7
Werkzeug==0.12.2
widgetsnbextension==3.0.2
egpbos commented 7 years ago

The most relevant packages in that pip list I guess are:

h5py==2.7.0
Keras==2.0.8
mcfly==1.0.1
numpy==1.13.1
protobuf==3.4.0
scikit-learn==0.19.0
tensorflow==1.3.0
wmkouw commented 7 years ago

On WSL (Windows Subsystem Linux), the same model load command gives:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-22-b568fbdcb1f4> in <module>()
----> 1 model = load_model('./model/model.h5')

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/models.pyc in load_model(filepath, custom_objects, compile)
    231             raise ValueError('No model found in config file.')
    232         model_config = json.loads(model_config.decode('utf-8'))
--> 233         model = model_from_config(model_config, custom_objects=custom_objects)
    234 
    235         # set weights

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/models.pyc in model_from_config(config, custom_objects)
    305                         'Maybe you meant to use '
    306                         '`Sequential.from_config(config)`?')
--> 307     return layer_module.deserialize(config, custom_objects=custom_objects)
    308 
    309 

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/layers/__init__.pyc in deserialize(config, custom_objects)
     52                                     module_objects=globs,
     53                                     custom_objects=custom_objects,
---> 54                                     printable_module_name='layer')

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/utils/generic_utils.pyc in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    137                 return cls.from_config(config['config'],
    138                                        custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 139                                                            list(custom_objects.items())))
    140             with CustomObjectScope(custom_objects):
    141                 return cls.from_config(config['config'])

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/models.pyc in from_config(cls, config, custom_objects)
   1207         model = cls()
   1208         for conf in config:
-> 1209             layer = layer_module.deserialize(conf, custom_objects=custom_objects)
   1210             model.add(layer)
   1211         return model

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/layers/__init__.pyc in deserialize(config, custom_objects)
     52                                     module_objects=globs,
     53                                     custom_objects=custom_objects,
---> 54                                     printable_module_name='layer')

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/utils/generic_utils.pyc in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name)
    137                 return cls.from_config(config['config'],
    138                                        custom_objects=dict(list(_GLOBAL_CUSTOM_OBJECTS.items()) +
--> 139                                                            list(custom_objects.items())))
    140             with CustomObjectScope(custom_objects):
    141                 return cls.from_config(config['config'])

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/layers/core.pyc in from_config(cls, config, custom_objects)
    692         elif function_type == 'lambda':
    693             # Unsafe deserialization from bytecode
--> 694             function = func_load(config['function'], globs=globs)
    695         else:
    696             raise TypeError('Unknown function type:', function_type)

/home/wmkouw/anaconda2/envs/mcfly/lib/python2.7/site-packages/keras/utils/generic_utils.pyc in func_load(code, defaults, closure, globs)
    198         if isinstance(defaults, list):
    199             defaults = tuple(defaults)
--> 200     code = marshal.loads(code.encode('raw_unicode_escape'))
    201     if globs is None:
    202         globs = globals()

ValueError: bad marshal data (unknown type code)

Other sources suggest bad marshal data is due to discrepancies in Python versions: https://github.com/fchollet/keras/issues/7440

conda version 4.3.25, python version 2.7.13

pip freeze:

backports-abc==0.5
backports.shutil-get-terminal-size==1.0.0
backports.ssl-match-hostname==3.5.0.1
backports.weakref==1.0rc1
bleach==1.5.0
certifi==2017.7.27.1
click==6.7
configparser==3.5.0
cycler==0.10.0
decorator==4.1.2
entrypoints==0.2.3
enum34==1.1.6
Flask==0.12.2
Flask-Cors==3.0.3
funcsigs==1.0.2
functools32==3.2.3.post2
gevent==1.2.2
greenlet==0.4.12
h5py==2.7.1
html5lib==0.9999999
ipykernel==4.6.1
ipython==5.5.0
ipython-genutils==0.2.0
ipywidgets==7.0.1
itsdangerous==0.24
Jinja2==2.9.6
jsonschema==2.6.0
jupyter==1.0.0
jupyter-client==5.1.0
jupyter-console==5.2.0
jupyter-core==4.3.0
Keras==2.0.6
Mako==1.0.7
Markdown==2.6.9
MarkupSafe==1.0
matplotlib==2.0.2
mcfly==1.0.1
mistune==0.7.4
mock==2.0.0
nbconvert==5.3.1
nbformat==4.4.0
notebook==5.1.0
numpy==1.13.1
olefile==0.44
pandas==0.20.3
pandocfilters==1.4.2
pathlib2==2.3.0
pbr==3.1.1
pexpect==4.2.1
pickleshare==0.7.4
Pillow==4.2.1
prompt-toolkit==1.0.15
protobuf==3.4.0
ptyprocess==0.5.2
Pygments==2.2.0
pygpu==0.7.1
pyparsing==2.2.0
python-dateutil==2.6.1
pytz==2017.2
PyYAML==3.12
pyzmq==16.0.2
qtconsole==4.3.1
quiver-engine==0.1.4.1.4
scandir==1.5
scikit-learn==0.19.0
scipy==0.19.1
simplegeneric==0.8.1
singledispatch==3.4.0.3
six==1.10.0
subprocess32==3.2.7
tensorflow==1.3.0
tensorflow-tensorboard==0.1.6
terminado==0.6
testpath==0.3.1
Theano==0.9.0
tornado==4.5.2
traitlets==4.3.2
wcwidth==0.1.7
webencodings==0.5
Werkzeug==0.12.2
widgetsnbextension==3.0.3
egpbos commented 7 years ago

Fixed my problem by reverting to Python 3.5, as suggested here: https://github.com/fchollet/keras/issues/

@cwmeijer: might be nice to add this to the documentation.

majthehero commented 6 years ago

@egpbos could you correct the link, it points to list of all issues now? I have the same problem. Thanks :)

egpbos commented 6 years ago

Sorry, I don't remember the full link, does reverting to Python 3.5 not work for you @majthehero ?

majthehero commented 6 years ago

It works 👍 I didn't notice that you wrote that, thanks.

florian-huber commented 5 years ago

It indeed seems to be inherent to Keras that models made+saved under python 3.5 often cannot be loaded under python 3.6. https://github.com/keras-team/keras/issues/9595 One workaround seems to be to save the architecture as code, and then only the weights in an h5 file.

ragapriy commented 2 years ago

ValueError Traceback (most recent call last)

in 17 #model = load_model('facenet_model/model.h5') 18 #return model ---> 19 model = keras.models.load_model('facenet_model/model.h5', custom_objects={'triplet_loss': triplet_loss}) ~\anaconda3\lib\site-packages\tensorflow\python\keras\saving\save.py in load_model(filepath, custom_objects, compile, options) 180 if (h5py is not None and ( 181 isinstance(filepath, h5py.File) or h5py.is_hdf5(filepath))): --> 182 return hdf5_format.load_model_from_hdf5(filepath, custom_objects, compile) 183 184 filepath = path_to_string(filepath) ~\anaconda3\lib\site-packages\tensorflow\python\keras\saving\hdf5_format.py in load_model_from_hdf5(filepath, custom_objects, compile) 175 raise ValueError('No model found in config file.') 176 model_config = json.loads(model_config.decode('utf-8')) --> 177 model = model_config_lib.model_from_config(model_config,custom_objects=custom_objects) 178 179 # set weights ~\anaconda3\lib\site-packages\tensorflow\python\keras\saving\model_config.py in model_from_config(config, custom_objects) 53 '`Sequential.from_config(config)`?') 54 from tensorflow.python.keras.layers import deserialize # pylint: disable=g-import-not-at-top ---> 55 return deserialize(config, custom_objects=custom_objects) 56 57 ~\anaconda3\lib\site-packages\tensorflow\python\keras\layers\serialization.py in deserialize(config, custom_objects) 169 """ 170 populate_deserializable_objects() --> 171 return generic_utils.deserialize_keras_object( 172 config, 173 module_objects=LOCAL.ALL_OBJECTS, ~\anaconda3\lib\site-packages\tensorflow\python\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name) 352 353 if 'custom_objects' in arg_spec.args: --> 354 return cls.from_config( 355 cls_config, 356 custom_objects=dict( ~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\training.py in from_config(cls, config, custom_objects) 2236 # be constructed for FunctionalModel 2237 from tensorflow.python.keras.engine import functional # pylint: disable=g-import-not-at-top -> 2238 return functional.Functional.from_config( 2239 config, custom_objects=custom_objects) 2240 ~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\functional.py in from_config(cls, config, custom_objects) 614 ValueError: In case of improperly formatted config dict. 615 """ --> 616 input_tensors, output_tensors, created_layers = reconstruct_from_config( 617 config, custom_objects) 618 model = cls(inputs=input_tensors, outputs=output_tensors, ~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\functional.py in reconstruct_from_config(config, custom_objects, created_layers) 1202 # First, we create all layers and enqueue nodes to be processed 1203 for layer_data in config['layers']: -> 1204 process_layer(layer_data) 1205 # Then we process nodes in order of layer depth. 1206 # Nodes that cannot yet be processed (if the inbound node ~\anaconda3\lib\site-packages\tensorflow\python\keras\engine\functional.py in process_layer(layer_data) 1184 from tensorflow.python.keras.layers import deserialize as deserialize_layer # pylint: disable=g-import-not-at-top 1185 -> 1186 layer = deserialize_layer(layer_data, custom_objects=custom_objects) 1187 created_layers[layer_name] = layer 1188 ~\anaconda3\lib\site-packages\tensorflow\python\keras\layers\serialization.py in deserialize(config, custom_objects) 169 """ 170 populate_deserializable_objects() --> 171 return generic_utils.deserialize_keras_object( 172 config, 173 module_objects=LOCAL.ALL_OBJECTS, ~\anaconda3\lib\site-packages\tensorflow\python\keras\utils\generic_utils.py in deserialize_keras_object(identifier, module_objects, custom_objects, printable_module_name) 352 353 if 'custom_objects' in arg_spec.args: --> 354 return cls.from_config( 355 cls_config, 356 custom_objects=dict( ~\anaconda3\lib\site-packages\tensorflow\python\keras\layers\core.py in from_config(cls, config, custom_objects) 1003 def from_config(cls, config, custom_objects=None): 1004 config = config.copy() -> 1005 function = cls._parse_function_from_config( 1006 config, custom_objects, 'function', 'module', 'function_type') 1007 ~\anaconda3\lib\site-packages\tensorflow\python\keras\layers\core.py in _parse_function_from_config(cls, config, custom_objects, func_attr_name, module_attr_name, func_type_attr_name) 1055 elif function_type == 'lambda': 1056 # Unsafe deserialization from bytecode -> 1057 function = generic_utils.func_load(config[func_attr_name], globs=globs) 1058 elif function_type == 'raw': 1059 function = config[func_attr_name] ~\anaconda3\lib\site-packages\tensorflow\python\keras\utils\generic_utils.py in func_load(code, defaults, closure, globs) 455 except (UnicodeEncodeError, binascii.Error): 456 raw_code = code.encode('raw_unicode_escape') --> 457 code = marshal.loads(raw_code) 458 if globs is None: 459 globs = globals() ValueError: bad marshal data (unknown type code) Im having this issue can anyone please tell how to solve this.