tf.summary.create_file_writer error in callback : not deep copyable

[lrosique]

When I try to launch a search with a callback using "create_file_writer" I get an error so I can't have pictures in tensorboard -> Images.

If I remove the line with create_file_writer and the "with...", the search does not raise any error but I don't get images (even with the tf.summary.image).

Code :

class CustomCallbackImage(Callback):
    def __init__(self, logpath):
        self.file_writer = tf.summary.create_file_writer(logpath + '/image')

    def on_epoch_end(self, epoch, logs):
        with self.file_writer.as_default():
            image1 = tf.random.uniform(shape=[8, 8, 1])
            tf.summary.image("grayscale_noise", [image1], step=epoch)

...
tuner.search(X_train, y_train,
          callbacks=[tbcb, CustomCallbackImage("logs")])

Stacktrace :

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
c:\workspaces\environnements\keras_gpu\lib\site-packages\keras_tuner\engine\tuner.py in _deepcopy_callbacks(self, callbacks)
    284         try:
--> 285             callbacks = copy.deepcopy(callbacks)
    286         except:

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_list(x, memo, deepcopy)
    214     for a in x:
--> 215         append(deepcopy(a, memo))
    216     return y

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 

C:\dev\python\python376\lib\copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_dict(x, memo, deepcopy)
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 

C:\dev\python\python376\lib\copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_dict(x, memo, deepcopy)
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 

C:\dev\python\python376\lib\copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_tuple(x, memo, deepcopy)
    219 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 220     y = [deepcopy(a, memo) for a in x]
    221     # We're not going to put the tuple in the memo, but it's still important we

C:\dev\python\python376\lib\copy.py in <listcomp>(.0)
    219 def _deepcopy_tuple(x, memo, deepcopy=deepcopy):
--> 220     y = [deepcopy(a, memo) for a in x]
    221     # We're not going to put the tuple in the memo, but it's still important we

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_dict(x, memo, deepcopy)
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    179                 else:
--> 180                     y = _reconstruct(x, memo, *rv)
    181 

C:\dev\python\python376\lib\copy.py in _reconstruct(x, memo, func, args, state, listiter, dictiter, deepcopy)
    279         if deep:
--> 280             state = deepcopy(state, memo)
    281         if hasattr(y, '__setstate__'):

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    149     if copier:
--> 150         y = copier(x, memo)
    151     else:

C:\dev\python\python376\lib\copy.py in _deepcopy_dict(x, memo, deepcopy)
    239     for key, value in x.items():
--> 240         y[deepcopy(key, memo)] = deepcopy(value, memo)
    241     return y

C:\dev\python\python376\lib\copy.py in deepcopy(x, memo, _nil)
    168                     if reductor:
--> 169                         rv = reductor(4)
    170                     else:

TypeError: can't pickle tensorflow.python._pywrap_tfe.EagerContextThreadLocalData objects

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
c:\workspaces\test_ia\ia_cnn_juillet\1- CNN custom TUNER.py in 
    135           verbose=1,
    136           validation_split=0.1,
--> 137           callbacks=[tbcb, checkpoint_cb, autostop, image_callback])
    138 
    139 best_model = tuner.get_best_models()[0]

c:\workspaces\environnements\keras_gpu\lib\site-packages\keras_tuner\engine\base_tuner.py in search(self, *fit_args, **fit_kwargs)
    142 
    143             self.on_trial_begin(trial)
--> 144             self.run_trial(trial, *fit_args, **fit_kwargs)
    145             self.on_trial_end(trial)
    146         self.on_search_end()

c:\workspaces\environnements\keras_gpu\lib\site-packages\keras_tuner\engine\multi_execution_tuner.py in run_trial(self, trial, *fit_args, **fit_kwargs)
     81         for execution in range(self.executions_per_trial):
     82             copied_fit_kwargs = copy.copy(fit_kwargs)
---> 83             callbacks = self._deepcopy_callbacks(original_callbacks)
     84             self._configure_tensorboard_dir(callbacks, trial, execution)
     85             callbacks.append(tuner_utils.TunerCallback(self, trial))

c:\workspaces\environnements\keras_gpu\lib\site-packages\keras_tuner\engine\tuner.py in _deepcopy_callbacks(self, callbacks)
    289                 "should be deep-copyable (since they are "
    290                 "reused across trials). "
--> 291                 "It is not possible to do `copy.deepcopy(%s)`" % (callbacks,)
    292             )
    293         return callbacks

ValueError: All callbacks used during a search should be deep-copyable (since they are reused across trials). It is not possible to do `copy.deepcopy([<tensorflow.python.keras.callbacks.TensorBoard object at 0x000001B872F36C48>, <tensorflow.python.keras.callbacks.ModelCheckpoint object at 0x000001B84B6A5F88>, <tensorflow.python.keras.callbacks.EarlyStopping object at 0x000001B84BB200C8>, <custom_callback.CustomCallbackImage object at 0x000001B84DF7D148>])`

[lrosique]

For HPARAMS in tensorboard integration i found this page in docs : https://keras.io/guides/keras_tuner/visualize_tuning/

[haifeng-jin]

KerasTuner try to deepcopy the callbacks so that when run multiple times of the training, the callbacks will always start fresh. Maybe we should try to rely on resetting the state instead of deep copying it.