perceptualJonathan
commented
4 years ago In looking at a few runs of this snippet, it looks like there is an off-by-1 error somewhere in the model loading procedure. Consistently, I see that if it is epoch X that is supposed to be loaded, and there is no checkpoint at epoch X, there is a checkpoint at epoch X+1. However, after account for that issue, I encounter a different error when trying to make predictions. If my test set is xTest, then calling
models = clf.tuner.get_best_models(MAXTRIALS)
for model in models:
model.predict(xTest)yields the error
---------------------------------------------------------------------------
UnimplementedError Traceback (most recent call last)
<ipython-input-5-c18752c5786e> in <module>
2
3 for model in models:
----> 4 model.predict(xTest)
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in _method_wrapper(self, *args, **kwargs)
128 raise ValueError('{} is not supported in multi-worker mode.'.format(
129 method.__name__))
--> 130 return method(self, *args, **kwargs)
131
132 return tf_decorator.make_decorator(
/usr/local/lib/python3.7/site-packages/tensorflow/python/keras/engine/training.py in predict(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)
1597 for step in data_handler.steps():
1598 callbacks.on_predict_batch_begin(step)
-> 1599 tmp_batch_outputs = predict_function(iterator)
1600 if data_handler.should_sync:
1601 context.async_wait()
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
778 else:
779 compiler = "nonXla"
--> 780 result = self._call(*args, **kwds)
781
782 new_tracing_count = self._get_tracing_count()
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
844 *args, **kwds)
845 # If we did not create any variables the trace we have is good enough.
--> 846 return self._concrete_stateful_fn._filtered_call(canon_args, canon_kwds) # pylint: disable=protected-access
847
848 def fn_with_cond(*inner_args, **inner_kwds):
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _filtered_call(self, args, kwargs, cancellation_manager)
1846 resource_variable_ops.BaseResourceVariable))],
1847 captured_inputs=self.captured_inputs,
-> 1848 cancellation_manager=cancellation_manager)
1849
1850 def _call_flat(self, args, captured_inputs, cancellation_manager=None):
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
1922 # No tape is watching; skip to running the function.
1923 return self._build_call_outputs(self._inference_function.call(
-> 1924 ctx, args, cancellation_manager=cancellation_manager))
1925 forward_backward = self._select_forward_and_backward_functions(
1926 args,
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/function.py in call(self, ctx, args, cancellation_manager)
548 inputs=args,
549 attrs=attrs,
--> 550 ctx=ctx)
551 else:
552 outputs = execute.execute_with_cancellation(
/usr/local/lib/python3.7/site-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
58 ctx.ensure_initialized()
59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60 inputs, attrs, num_outputs)
61 except core._NotOkStatusException as e:
62 if name is not None:
UnimplementedError: Cast double to string is not supported
[[node functional_1/Cast (defined at <ipython-input-5-c18752c5786e>:4) ]] [Op:__inference_predict_function_9685]
Function call stack:
predict_function
stale[bot]
haifeng-jin
Bug Description
Bug Reproduction
Code for reproducing the bug:
Data used by the code:
Expected Behavior
Setup Details
Include the details about the versions of:
Additional context