[17_deep_learning]
02_how_to_use_tensorflow
6 Train Model
when run the code:
training=model.fit(X,
Y,
epochs=50,
validation_split=.2,
batch_size=128,
verbose=1,
callbacks=[tb_callback])
encountered an error：
Epoch 1/50
2/313 [..............................] - ETA: 1:32 - loss: nan - accuracy: 0.4648WARNING:tensorflow:Callbacks method on_train_batch_begin is slow compared to the batch time (batch time: 0.0020s vs on_train_batch_begin time: 0.0070s). Check your callbacks.
WARNING:tensorflow:Callbacks method on_train_batch_end is slow compared to the batch time (batch time: 0.0020s vs on_train_batch_end time: 0.5843s). Check your callbacks.
306/313 [============================>.] - ETA: 0s - loss: nan - accuracy: 0.4987
InvalidArgumentError Traceback (most recent call last)
Input In [25], in <cell line: 1>()
----> 1 training=model.fit(X,
2 Y,
3 epochs=50,
4 validation_split=.2,
5 batch_size=128,
6 verbose=1,
7 callbacks=[tb_callback])
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:108, in enable_multi_worker.._method_wrapper(self, *args, kwargs)
106 def _method_wrapper(self, *args, *kwargs):
107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
--> 108 return method(self, args, kwargs)
110 # Running inside run_distribute_coordinator already.
111 if dc_context.get_current_worker_context():
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:1137, in Model.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)
1134 vallogs = {'val' + name: val for name, val in val_logs.items()}
1135 epoch_logs.update(val_logs)
-> 1137 callbacks.on_epoch_end(epoch, epoch_logs)
1138 training_logs = epoch_logs
1139 if self.stop_training:
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:412, in CallbackList.on_epoch_end(self, epoch, logs)
410 for callback in self.callbacks:
411 if getattr(callback, '_supports_tf_logs', False):
--> 412 callback.on_epoch_end(epoch, logs)
413 else:
414 if numpy_logs is None: # Only convert once.
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2182, in TensorBoard.on_epoch_end(self, epoch, logs)
2179 self._log_epoch_metrics(epoch, logs)
2181 if self.histogram_freq and epoch % self.histogram_freq == 0:
-> 2182 self._log_weights(epoch)
2184 if self.embeddings_freq and epoch % self.embeddings_freq == 0:
2185 self._log_embeddings(epoch)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2234, in TensorBoard._log_weights(self, epoch)
2232 for weight in layer.weights:
2233 weightname = weight.name.replace(':', '')
-> 2234 summary_ops_v2.histogram(weight_name, weight, step=epoch)
2235 if self.write_images:
2236 self._log_weight_as_image(weight, weight_name, epoch)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:836, in histogram(name, tensor, family, step)
827 def function(tag, scope):
828 # Note the identity to move the tensor to the CPU.
829 return gen_summary_ops.write_histogram_summary(
830 _summary_state.writer._resource, # pylint: disable=protected-access
831 _choose_step(step),
832 tag,
833 array_ops.identity(tensor),
834 name=scope)
--> 836 return summary_writer_function(name, tensor, function, family=family)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:764, in summary_writer_function(name, tensor, function, family)
762 return control_flow_ops.no_op()
763 with ops.device("cpu:0"):
--> 764 op = smart_cond.smart_cond(
765 should_record_summaries(), record, _nothing, name="")
766 if not context.executing_eagerly():
767 ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\framework\smart_cond.py:54, in smart_cond(pred, true_fn, false_fn, name)
52 if pred_value is not None:
53 if pred_value:
---> 54 return true_fn()
55 else:
56 return false_fn()
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:758, in summary_writer_function..record()
755 def record():
756 with ops.name_scope(name_scope), summary_op_util.summary_scope(
757 name, family, values=[tensor]) as (tag, scope):
--> 758 with ops.control_dependencies([function(tag, scope)]):
759 return constant_op.constant(True)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:829, in histogram..function(tag, scope)
827 def function(tag, scope):
828 # Note the identity to move the tensor to the CPU.
--> 829 return gen_summary_ops.write_histogram_summary(
830 _summary_state.writer._resource, # pylint: disable=protected-access
831 _choose_step(step),
832 tag,
833 array_ops.identity(tensor),
834 name=scope)
[17_deep_learning]
02_how_to_use_tensorflow
6 Train Model
when run the code:
training=model.fit(X,
Y, epochs=50, validation_split=.2, batch_size=128, verbose=1, callbacks=[tb_callback])
encountered an error：
Epoch 1/50 2/313 [..............................] - ETA: 1:32 - loss: nan - accuracy: 0.4648WARNING:tensorflow:Callbacks method
on_train_batch_beginis slow compared to the batch time (batch time: 0.0020s vson_train_batch_begintime: 0.0070s). Check your callbacks. WARNING:tensorflow:Callbacks methodon_train_batch_endis slow compared to the batch time (batch time: 0.0020s vson_train_batch_endtime: 0.5843s). Check your callbacks. 306/313 [============================>.] - ETA: 0s - loss: nan - accuracy: 0.4987InvalidArgumentError Traceback (most recent call last) Input In [25], in <cell line: 1>() ----> 1 training=model.fit(X,
2 Y, 3 epochs=50, 4 validation_split=.2, 5 batch_size=128, 6 verbose=1, 7 callbacks=[tb_callback])
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:108, in enable_multi_worker.._method_wrapper(self, *args, kwargs)
106 def _method_wrapper(self, *args, *kwargs):
107 if not self._in_multi_worker_mode(): # pylint: disable=protected-access
--> 108 return method(self, args, kwargs)
110 # Running inside
run_distribute_coordinatoralready. 111 if dc_context.get_current_worker_context():File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\engine\training.py:1137, in Model.fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing) 1134 vallogs = {'val' + name: val for name, val in val_logs.items()} 1135 epoch_logs.update(val_logs) -> 1137 callbacks.on_epoch_end(epoch, epoch_logs) 1138 training_logs = epoch_logs 1139 if self.stop_training:
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:412, in CallbackList.on_epoch_end(self, epoch, logs) 410 for callback in self.callbacks: 411 if getattr(callback, '_supports_tf_logs', False): --> 412 callback.on_epoch_end(epoch, logs) 413 else: 414 if numpy_logs is None: # Only convert once.
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2182, in TensorBoard.on_epoch_end(self, epoch, logs) 2179 self._log_epoch_metrics(epoch, logs) 2181 if self.histogram_freq and epoch % self.histogram_freq == 0: -> 2182 self._log_weights(epoch) 2184 if self.embeddings_freq and epoch % self.embeddings_freq == 0: 2185 self._log_embeddings(epoch)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\keras\callbacks.py:2234, in TensorBoard._log_weights(self, epoch) 2232 for weight in layer.weights: 2233 weightname = weight.name.replace(':', '') -> 2234 summary_ops_v2.histogram(weight_name, weight, step=epoch) 2235 if self.write_images: 2236 self._log_weight_as_image(weight, weight_name, epoch)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:836, in histogram(name, tensor, family, step) 827 def function(tag, scope): 828 # Note the identity to move the tensor to the CPU. 829 return gen_summary_ops.write_histogram_summary( 830 _summary_state.writer._resource, # pylint: disable=protected-access 831 _choose_step(step), 832 tag, 833 array_ops.identity(tensor), 834 name=scope) --> 836 return summary_writer_function(name, tensor, function, family=family)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:764, in summary_writer_function(name, tensor, function, family) 762 return control_flow_ops.no_op() 763 with ops.device("cpu:0"): --> 764 op = smart_cond.smart_cond( 765 should_record_summaries(), record, _nothing, name="") 766 if not context.executing_eagerly(): 767 ops.add_to_collection(ops.GraphKeys._SUMMARY_COLLECTION, op) # pylint: disable=protected-access
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\framework\smart_cond.py:54, in smart_cond(pred, true_fn, false_fn, name) 52 if pred_value is not None: 53 if pred_value: ---> 54 return true_fn() 55 else: 56 return false_fn()
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:758, in summary_writer_function..record()
755 def record():
756 with ops.name_scope(name_scope), summary_op_util.summary_scope(
757 name, family, values=[tensor]) as (tag, scope):
--> 758 with ops.control_dependencies([function(tag, scope)]):
759 return constant_op.constant(True)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\summary_ops_v2.py:829, in histogram..function(tag, scope)
827 def function(tag, scope):
828 # Note the identity to move the tensor to the CPU.
--> 829 return gen_summary_ops.write_histogram_summary(
830 _summary_state.writer._resource, # pylint: disable=protected-access
831 _choose_step(step),
832 tag,
833 array_ops.identity(tensor),
834 name=scope)
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\gen_summary_ops.py:478, in write_histogram_summary(writer, step, tag, values, name) 476 pass 477 try: --> 478 return write_histogram_summary_eager_fallback( 479 writer, step, tag, values, name=name, ctx=_ctx) 480 except _core._SymbolicException: 481 pass # Add nodes to the TensorFlow graph.
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\ops\gen_summary_ops.py:497, in write_histogram_summary_eager_fallback(writer, step, tag, values, name, ctx) 495 _inputs_flat = [writer, step, tag, values] 496 _attrs = ("T", _attr_T) --> 497 _result = _execute.execute(b"WriteHistogramSummary", 0, inputs=_inputs_flat, 498 attrs=_attrs, ctx=ctx, name=name) 499 _result = None 500 return _result
File F:\Python\Miniconda3\envs\ml4t\lib\site-packages\tensorflow\python\eager\execute.py:59, in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name) 57 try: 58 ctx.ensure_initialized() ---> 59 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name, 60 inputs, attrs, num_outputs) 61 except core._NotOkStatusException as e: 62 if name is not None:
InvalidArgumentError: Nan in summary histogram for: hidden/kernel_0 [Op:WriteHistogramSummary]