add examples for embeddings case to docs

[PiercarloSlavazza]

Could you please add the example or at least give some hint about how to build the model?

I - naively - tried with:

model = Sequential()
model.add(tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
model.add(tf.keras.layers.GlobalAveragePooling1D())
model.add(params['first_neuron'], activation=params['activation'])
model.add(tf.keras.layers.Dense(len(classes) + 1, activation='softmax'))
model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])

and got the error:

TypeError: The added layer must be an instance of class Layer. Found: <tensorflow.python.keras.layers.embeddings.Embedding object at 0x000001EBD4844D48>

but I have no clue about a possible workaround.

Thank you.

[PiercarloSlavazza]

I have now understood that I have NOT to mix tf.keras and keras.

So, I currently have:

params = {
    'first_neuron': [100, 200, 300],
    'optimizer': ['Nadam', 'Adam']
}

model = Sequential()
model.add(keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(params['first_neuron'], activation='relu'))

which gives the error:

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-36-750a3f6055be> in <module>
      2 model.add(keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
      3 model.add(keras.layers.GlobalAveragePooling1D())
----> 4 model.add(keras.layers.Dense(params['first_neuron'], activation='relu'))
      5 #model.add(keras.layers.Dense(len(classes) + 1, activation='softmax'))
      6 #model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\engine\sequential.py in add(self, layer)
    179                 self.inputs = network.get_source_inputs(self.outputs[0])
    180         elif self.outputs:
--> 181             output_tensor = layer(self.outputs[0])
    182             if isinstance(output_tensor, list):
    183                 raise TypeError('All layers in a Sequential model '

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\engine\base_layer.py in __call__(self, inputs, **kwargs)
    429                                          'You can build it manually via: '
    430                                          '`layer.build(batch_input_shape)`')
--> 431                 self.build(unpack_singleton(input_shapes))
    432                 self.built = True
    433 

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\layers\core.py in build(self, input_shape)
    864                                       name='kernel',
    865                                       regularizer=self.kernel_regularizer,
--> 866                                       constraint=self.kernel_constraint)
    867         if self.use_bias:
    868             self.bias = self.add_weight(shape=(self.units,),

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\legacy\interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\engine\base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint)
    247         if dtype is None:
    248             dtype = K.floatx()
--> 249         weight = K.variable(initializer(shape),
    250                             dtype=dtype,
    251                             name=name,

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\initializers.py in __call__(self, shape, dtype)
    207             scale /= max(1., fan_out)
    208         else:
--> 209             scale /= max(1., float(fan_in + fan_out) / 2)
    210         if self.distribution == 'normal':
    211             # 0.879... = scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)

TypeError: unsupported operand type(s) for +: 'int' and 'list'

[PiercarloSlavazza]

Sorry, totally on the wrong track :-(

I now also understand that I have to pass a function to scan - and not a model (makes sense, of course).

So, one step closer, I now have:

p = {
    'first_neuron': [100, 200, 300],
    'optimizer': ['nadam', 'adam']
}

def nlp_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
    model.add(keras.layers.GlobalAveragePooling1D())
    model.add(keras.layers.Dense(params['first_neuron'], activation='relu'))
    model.add(keras.layers.Dense(len(classes) + 1, activation='softmax'))
    model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])

    out = model.fit(x_train, y_train, validation_data=[x_val, y_val], verbose=0)

    return out, model

ta.Scan(training_padded, training_classes_seq, model=nlp_model, params=p, grid_downsample=0.1)

which gives:

---------------------------------------------------------------------------
TalosDataError                            Traceback (most recent call last)
<ipython-input-51-2402648ee6e6> in <module>
     17     return out, model
     18 
---> 19 ta.Scan(training_padded, training_classes_seq, model=nlp_model, params=p, grid_downsample=0.1)
     20 
     21 #model = Sequential()

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\Scan.py in __init__(self, x, y, params, model, dataset_name, experiment_no, experiment_name, x_val, y_val, val_split, shuffle, round_limit, time_limit, grid_downsample, random_method, seed, search_method, permutation_filter, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, reduce_loss, last_epoch_value, clear_tf_session, disable_progress_bar, print_params, debug)
    183         # input parameters section ends
    184 
--> 185         self._null = self.runtime()
    186 
    187     def runtime(self):

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\Scan.py in runtime(self)
    187     def runtime(self):
    188 
--> 189         self = scan_prepare(self)
    190         self = scan_run(self)

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\scan_prepare.py in scan_prepare(self)
     42 
     43     # create the paramater object and move to self
---> 44     self.paramgrid_object = ParamGrid(self)
     45     self.param_log = self.paramgrid_object.param_log
     46     self.param_grid = self.paramgrid_object.param_grid

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\parameters\ParamGrid.py in __init__(self, main_self)
     47         self.param_grid = self._create_param_grid(ls,
     48                                                   final_grid_size,
---> 49                                                   virtual_grid_size)
     50 
     51         # handle the case where permutation filter is provided

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\parameters\ParamGrid.py in _create_param_grid(self, ls, final_grid_size, virtual_grid_size)
     70         # select permutations according to downsample
     71         if final_grid_size < virtual_grid_size:
---> 72             out = sample_reducer(self, final_grid_size, virtual_grid_size)
     73         else:
     74             out = range(0, final_grid_size)

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\reducers\sample_reducer.py in sample_reducer(self, length, max_value)
     31     # throw an error if
     32     if n < 1:
---> 33         raise TalosDataError("No permutations in grid. Incease grid_downsample")
     34 
     35     # Initialize Randomizer()

TalosDataError: No permutations in grid. Incease grid_downsample

[PiercarloSlavazza]

Fixed another couple of glitches, and now I have:

p = {
    'first_neuron': [100, 200, 300],
    'optimizer': ['nadam', 'adam'],
    'epochs': [10,20]
}

def nlp_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
    model.add(keras.layers.GlobalAveragePooling1D())
    model.add(keras.layers.Dense(params['first_neuron'], activation='relu'))
    model.add(keras.layers.Dense(len(classes) + 1, activation='softmax'))
    model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])

    out = model.fit(x_train, y_train, validation_data=[x_val, y_val], epochs=params['epochs'], verbose=0)

    return out, model

ta.Scan(training_padded, training_classes_seq, model=nlp_model, params=p, grid_downsample=0.5)

which unfortunately gives:

---------------------------------------------------------------------------
InternalError                             Traceback (most recent call last)
C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
   1355     try:
-> 1356       return fn(*args)
   1357     except errors.OpError as e:

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
   1340       return self._call_tf_sessionrun(
-> 1341           options, feed_dict, fetch_list, target_list, run_metadata)
   1342 

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
   1428         self._session, options, feed_dict, fetch_list, target_list,
-> 1429         run_metadata)
   1430 

InternalError: GPU sync failed

During handling of the above exception, another exception occurred:

InternalError                             Traceback (most recent call last)
<ipython-input-54-6bee7a6314b7> in <module>
     18     return out, model
     19 
---> 20 ta.Scan(training_padded, training_classes_seq, model=nlp_model, params=p, grid_downsample=0.5)
     21 
     22 #model = Sequential()

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\Scan.py in __init__(self, x, y, params, model, dataset_name, experiment_no, experiment_name, x_val, y_val, val_split, shuffle, round_limit, time_limit, grid_downsample, random_method, seed, search_method, permutation_filter, reduction_method, reduction_interval, reduction_window, reduction_threshold, reduction_metric, reduce_loss, last_epoch_value, clear_tf_session, disable_progress_bar, print_params, debug)
    183         # input parameters section ends
    184 
--> 185         self._null = self.runtime()
    186 
    187     def runtime(self):

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\Scan.py in runtime(self)
    188 
    189         self = scan_prepare(self)
--> 190         self = scan_run(self)

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\scan_run.py in scan_run(self)
     18     # start the main loop of the program
     19     while len(self.param_log) != 0:
---> 20         self = scan_round(self)
     21         self.pbar.update(1)
     22         if self.time_limit is not None:

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\scan\scan_round.py in scan_round(self)
     30     # fit the model
     31     try:
---> 32         _hr_out, self.keras_model = ingest_model(self)
     33     except TypeError as err:
     34         if err.args[0] == "unsupported operand type(s) for +: 'int' and 'numpy.str_'":

C:\tools\Anaconda3\envs\talos\lib\site-packages\talos\model\ingest_model.py in ingest_model(self)
      8                       self.x_val,
      9                       self.y_val,
---> 10                       self.round_params)

<ipython-input-54-6bee7a6314b7> in nlp_model(x_train, y_train, x_val, y_val, params)
     14     model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])
     15 
---> 16     out = model.fit(x_train, y_train, validation_data=[x_val, y_val], epochs=params['epochs'], verbose=0)
     17 
     18     return out, model

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\engine\training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
   1037                                         initial_epoch=initial_epoch,
   1038                                         steps_per_epoch=steps_per_epoch,
-> 1039                                         validation_steps=validation_steps)
   1040 
   1041     def evaluate(self, x=None, y=None,

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\engine\training_arrays.py in fit_loop(model, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
    197                     ins_batch[i] = ins_batch[i].toarray()
    198 
--> 199                 outs = f(ins_batch)
    200                 outs = to_list(outs)
    201                 for l, o in zip(out_labels, outs):

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\backend\tensorflow_backend.py in __call__(self, inputs)
   2695 
   2696     def __call__(self, inputs):
-> 2697         if hasattr(get_session(), '_make_callable_from_options'):
   2698             if py_any(is_sparse(x) for x in self.inputs):
   2699                 if py_any(is_tensor(x) for x in inputs):

C:\tools\Anaconda3\envs\talos\lib\site-packages\keras\backend\tensorflow_backend.py in get_session()
    197                 # not already marked as initialized.
    198                 is_initialized = session.run(
--> 199                     [tf.is_variable_initialized(v) for v in candidate_vars])
    200                 uninitialized_vars = []
    201                 for flag, v in zip(is_initialized, candidate_vars):

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in run(self, fetches, feed_dict, options, run_metadata)
    948     try:
    949       result = self._run(None, fetches, feed_dict, options_ptr,
--> 950                          run_metadata_ptr)
    951       if run_metadata:
    952         proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
   1171     if final_fetches or final_targets or (handle and feed_dict_tensor):
   1172       results = self._do_run(handle, final_targets, final_fetches,
-> 1173                              feed_dict_tensor, options, run_metadata)
   1174     else:
   1175       results = []

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
   1348     if handle is None:
   1349       return self._do_call(_run_fn, feeds, fetches, targets, options,
-> 1350                            run_metadata)
   1351     else:
   1352       return self._do_call(_prun_fn, handle, feeds, fetches)

C:\tools\Anaconda3\envs\talos\lib\site-packages\tensorflow\python\client\session.py in _do_call(self, fn, *args)
   1368           pass
   1369       message = error_interpolation.interpolate(message, self._graph)
-> 1370       raise type(e)(node_def, op, message)
   1371 
   1372   def _extend_graph(self):

InternalError: GPU sync failed

InternalError: GPU sync failed: I am using tensorflow-gpu (installed via conda) - is it a known limitation that this is incompatible with talos, or I made some other mistake?

[PiercarloSlavazza]

Finally got it working:

For the records:

• conda installation (don't know if I missed it, but you should add it somewhere given that currently there are some dependencies issues, plus bugs w/ h5py 2.9):

conda create --name talos scipy=1.2.0 matplotlib=2.2.3  h5py=2.8.0 tensorflow jupyter
conda activate talos
pip install talos

• talos/keras code:

p = {
    'first_neuron': [100, 200],
    'optimizer': ['nadam', 'adam'],
    'epochs': [10]
}

def nlp_model(x_train, y_train, x_val, y_val, params):

    model = Sequential()
    model.add(keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length))
    model.add(keras.layers.GlobalAveragePooling1D())
    model.add(keras.layers.Dense(params['first_neuron'], activation='relu'))
    model.add(keras.layers.Dense(len(classes) + 1, activation='softmax'))
    model.compile(loss='sparse_categorical_crossentropy', optimizer=params['optimizer'], metrics=['accuracy'])

    out = model.fit(x_train, y_train, validation_data=[x_val, y_val], epochs=params['epochs'], verbose=0)

    return out, model

h = ta.Scan(training_padded, training_classes_seq, model=nlp_model, params=p, grid_downsample=0.5)

Of course w/o GPU it is dead slow. I will try again - and report here if I will succeed.

[PiercarloSlavazza]

Of course w/o GPU it is dead slow. I will try again - and report here if I will succeed.

With this installation command, everything went well and execution is super fast again:

conda create --name talos_gpu scipy=1.2.0 matplotlib=2.2.3  h5py=2.8.0 tensorflow-gpu jupyter
conda activate talos_gpu
pip install talos